jsonlite

SEXP C_collapse_array(SEXP x) {
  if (!isString(x))
    error("x must be a character vector.");

  int len = length(x);
  size_t nchar_total = 0;

  for (int i=0; i<len; i++) {
    nchar_total += strlen(translateCharUTF8(STRING_ELT(x, i)));
  }

  char *s = malloc(nchar_total+len+3); //if len is 0, we need at least: '[]\0'
  char *olds = s;
  size_t size;

  for (int i=0; i<len; i++) {
    s[0] = ',';
    size = strlen(translateCharUTF8(STRING_ELT(x, i)));
    memcpy(++s, translateCharUTF8(STRING_ELT(x, i)), size);
    s += size;
  }
  if(olds == s) s++;
  olds[0] = '[';
  s[0] = ']';
  s[1] = '\0';

  //get character encoding from first element
  SEXP out = PROTECT(allocVector(STRSXP, 1));
  SET_STRING_ELT(out, 0, mkCharCE(olds,  CE_UTF8));
  UNPROTECT(1);
  free(olds);
  return out;
}

SEXP C_collapse_array_pretty_inner(SEXP x) {
  if (!isString(x))
    error("x must character vector.");

  //calculate required space
  size_t len = Rf_length(x);
  size_t nchar_total = 0;
  for (int i=0; i<len; i++) {
    nchar_total += strlen(translateCharUTF8(STRING_ELT(x, i)));
  }

  // n-1 ", " separators
  if(len){
    nchar_total += (len-1)*2;
  }

  //outer parentheses plus terminator
  nchar_total += 3;

  //allocate memory and create a cursor
  char *str = malloc(nchar_total);
  char *cursor = str;
  char **cur = &cursor;

  //init object
  append_text(cur, "[", 1);

  //copy everything
  for (int i=0; i<len; i++) {
    append_text(cur, translateCharUTF8(STRING_ELT(x, i)), -1);
    append_text(cur, ", ", 2);
  }

  //remove trailing ", "
  if(len) {
    cursor -= 2;
  }

  //finish up
  append_text(cur, "]\0", 2);

  //encode as UTF8 string
  SEXP out = PROTECT(allocVector(STRSXP, 1));
  SET_STRING_ELT(out, 0, mkCharCE(str,  CE_UTF8));
  UNPROTECT(1);
  free(str);
  return out;
}

SEXP C_escape_chars(SEXP x) {
  if (!isString(x))
    error("x must be a character vector.");
  if (x == R_NilValue || length(x) == 0)
    return x;

  int len = length(x);
  SEXP out = PROTECT(allocVector(STRSXP, len));

  for (int i=0; i<len; i++) {
    SET_STRING_ELT(out, i, C_escape_chars_one(STRING_ELT(x, i)));
  }
  UNPROTECT(1);
  return out;
}

SEXP C_escape_chars_one(SEXP x) {

  // Make a cursor pointer
  const char * cur = CHAR(x);
  const char * end = CHAR(x) + Rf_length(x);

  // Count the number of matches
  int matches = 0;
  while (cur < end) {
    switch(*cur) {
      case '\\':
      case '"':
      case '\n':
      case '\r':
      case '\t':
      case '\b':
      case '\f':
        matches++;
        break;
      case '/':
        if(cur > CHAR(x) && cur[-1] == '<')
          matches++;
        break;
      default:
        if (*cur >= 0x00 && *cur <= 0x1f)
          matches += 5; //needs explicit \u00xx escaping
    }
    cur++;
  }

  // Calculate output length, 2 for double quotes
  size_t outlen = Rf_length(x) + matches + 2;
  char * newstr = malloc(outlen);

  // Reset cursor to beginning
  cur = CHAR(x);

  // Allocate string memory; add 2 for start and end quotes.
  char * outcur = newstr;
  *outcur++ = '"';

  while(cur < end) {
    switch(*cur) {
      case '\\':
        *outcur++ = '\\';
        *outcur = '\\';
        break;
      case '"':
        *outcur++ = '\\';
        *outcur = '"';
        break;
      case '\n':
        *outcur++ = '\\';
        *outcur = 'n';
        break;
      case '\r':
        *outcur++ = '\\';
        *outcur = 'r';
        break;
      case '\t':
        *outcur++ = '\\';
        *outcur = 't';
        break;
      case '\b':
        *outcur++ = '\\';
        *outcur = 'b';
        break;
      case '\f':
        *outcur++ = '\\';
        *outcur = 'f';
        break;
      case '/':
        if(cur > CHAR(x) && cur[-1] == '<'){
          *outcur++ = '\\';
          *outcur = '/';
          break;
        } //FALL THROUGH!
      default:
        //control characters need explicit \u00xx escaping
        if (*cur >= 0x00 && *cur <= 0x1f){
          snprintf(outcur, 7, "\\u%04x", *cur);
          outcur += 5; //extra length
          break;
        }
        //simply copy char from input
        *outcur = *cur;
    }

    //increment input and output cursors to next character
    cur++;
    outcur++;
  }

  //Close quote and create R string
  *outcur = '"';
  SEXP out = mkCharLenCE(newstr, outlen, getCharCE(x));
  free(newstr);
  return out;
}

SEXP C_is_datelist(SEXP x) {
  size_t len = Rf_length(x);
  if(!Rf_isVectorList(x) || len == 0)
    return ScalarLogical(FALSE);

  // Need
  int status = FALSE;
  for (size_t i = 0; i < len; i++) {
    SEXP el = VECTOR_ELT(x, i);
    if(Rf_isNull(el))
      continue;
    if(Rf_isString(el) && Rf_length(el) > 0 && !strcmp(CHAR(STRING_ELT(el, 0)), "NA"))
      continue;
    if(Rf_isNumeric(el) && Rf_inherits(el, "POSIXct")){
      status = TRUE; //at least one date
    } else {
      return ScalarLogical(FALSE); // quit immediately
    }
  }

  return ScalarLogical(status);
}

SEXP C_is_recordlist(SEXP x){
  return ScalarLogical(is_recordlist(x));
}

SEXP C_is_scalarlist(SEXP x) {

  bool is_scalarlist = true;
  if (TYPEOF(x) != VECSXP){
    is_scalarlist = false;
  } else {
    SEXP el;
    int len = length(x);
    for (int i=0; i<len; i++) {
      el = VECTOR_ELT(x, i);
      switch(TYPEOF(el)) {
        case LGLSXP:
        case INTSXP:
        case REALSXP:
        case STRSXP:
        case NILSXP:
        case RAWSXP: //not used but for compatibility with is.atomic
        case CPLXSXP: //not used but for compatibility with is.atomic
          if(length(el) < 2) continue;
          //else fall through
        default:
          is_scalarlist = false;
          break;
      }
    }
  }

  //get character encoding from first element
  return ScalarLogical(is_scalarlist);
}

static SEXP ParseArray(yajl_val node, int bigint){
  int len = YAJL_GET_ARRAY(node)->len;
  SEXP vec = PROTECT(allocVector(VECSXP, len));
  for (int i = 0; i < len; ++i) {
    SET_VECTOR_ELT(vec, i, ParseValue(YAJL_GET_ARRAY(node)->values[i], bigint));
  }
  UNPROTECT(1);
  return vec;
}

static SEXP ParseObject(yajl_val node, int bigint){
  int len = YAJL_GET_OBJECT(node)->len;
  SEXP keys = PROTECT(allocVector(STRSXP, len));
  SEXP vec = PROTECT(allocVector(VECSXP, len));
  for (int i = 0; i < len; ++i) {
    SET_STRING_ELT(keys, i, mkCharCE(YAJL_GET_OBJECT(node)->keys[i], CE_UTF8));
    SET_VECTOR_ELT(vec, i, ParseValue(YAJL_GET_OBJECT(node)->values[i], bigint));
  }
  setAttrib(vec, R_NamesSymbol, keys);
  UNPROTECT(2);
  return vec;
}

SEXP ParseValue(yajl_val node, int bigint){
  if(YAJL_IS_NULL(node)){
    return R_NilValue;
  }
  if(YAJL_IS_STRING(node)){
    SEXP tmp = PROTECT(allocVector(STRSXP, 1));
    SET_STRING_ELT(tmp, 0, mkCharCE(YAJL_GET_STRING(node),  CE_UTF8));
    UNPROTECT(1);
    return tmp;
  }
  if(YAJL_IS_INTEGER(node)){
    long long int val = YAJL_GET_INTEGER(node);
    /* 2^53 is highest int stored as double without loss */
    if(bigint && (val > 9007199254740992 || val < -9007199254740992)){
      char buf[32];
      snprintf(buf, 32, "%lld", val);
      return mkString(buf);
    /* see .Machine$integer.max in R */
    } else if(val > 2147483647 || val < -2147483647){
      return ScalarReal(val);
    } else {
      return ScalarInteger(val);
    }
  }
  if(YAJL_IS_DOUBLE(node)){
    return(ScalarReal(YAJL_GET_DOUBLE(node)));
  }
  if(YAJL_IS_NUMBER(node)){
    /* A number that is not int or double (very rare) */
    /* This seems to correctly round to Inf/0/-Inf */
    return(ScalarReal(YAJL_GET_DOUBLE(node)));
  }
  if(YAJL_IS_TRUE(node)){
    return(ScalarLogical(1));
  }
  if(YAJL_IS_FALSE(node)){
    return(ScalarLogical(0));
  }
  if(YAJL_IS_OBJECT(node)){
    return(ParseObject(node, bigint));
  }
  if(YAJL_IS_ARRAY(node)){
    return(ParseArray(node, bigint));
  }
  error("Invalid YAJL node type.");
}

SEXP R_integer64_to_char(SEXP x, SEXP na_as_string){
  int len = length(x);
  int na_string = asLogical(na_as_string);
  long long * xint = (long long *) REAL(x);
  char buf[32];
  SEXP out = PROTECT(allocVector(STRSXP, len));
  for (int i = 0; i < len; i++) {
    if(xint[i] == NA_INTEGER64){
      if(na_string == NA_LOGICAL){
        SET_STRING_ELT(out, i, NA_STRING);
      } else if(na_string){
        SET_STRING_ELT(out, i, mkChar("\"NA\""));
      } else {
        SET_STRING_ELT(out, i, mkChar("null"));
      }
    } else {
      #ifdef _WIN32
        snprintf(buf, 32, "%lld", xint[i]);
      #else
        //snprintf(buf, 32, "%lld", xint[i]);
        //modp is faster (but does not work on windows)
        modp_litoa10(xint[i], buf);
      #endif
      SET_STRING_ELT(out, i, mkChar(buf));
    }
  }
  UNPROTECT(1);
  return out;
}

SEXP R_num_to_char(SEXP x, SEXP digits, SEXP na_as_string, SEXP use_signif, SEXP always_decimal) {
  int len = length(x);
  int na_string = asLogical(na_as_string);
  int signif = asLogical(use_signif);
  int always_dec = asLogical(always_decimal);
  char buf[32];
  SEXP out = PROTECT(allocVector(STRSXP, len));
  if(isInteger(x)){
    for (int i=0; i<len; i++) {
      if(INTEGER(x)[i] == NA_INTEGER){
        if(na_string == NA_LOGICAL){
          SET_STRING_ELT(out, i, NA_STRING);
        } else if(na_string){
          SET_STRING_ELT(out, i, mkChar("\"NA\""));
        } else {
          SET_STRING_ELT(out, i, mkChar("null"));
        }
      } else {
        modp_itoa10(INTEGER(x)[i], buf);
        SET_STRING_ELT(out, i, mkChar(buf));
      }
    }
  } else if(isReal(x)) {
    int precision = asInteger(digits);
    int sig_digits = signif ? ceil(fmin(17, precision)) : 0;
    double * xreal = REAL(x);
    for (int i=0; i<len; i++) {
      double val = xreal[i];
      if(!R_FINITE(val)){
        if(na_string == NA_LOGICAL){
          SET_STRING_ELT(out, i, NA_STRING);
        } else if(na_string){
          if(ISNA(val)){
            SET_STRING_ELT(out, i, mkChar("\"NA\""));
          } else if(ISNAN(val)){
            SET_STRING_ELT(out, i, mkChar("\"NaN\""));
          } else if(val == R_PosInf){
            SET_STRING_ELT(out, i, mkChar("\"Inf\""));
          } else if(val == R_NegInf){
            SET_STRING_ELT(out, i, mkChar("\"-Inf\""));
          } else {
            error("Unrecognized non finite value.");
          }
        } else {
          SET_STRING_ELT(out, i, mkChar("null"));
        }
      } else {
        if(precision == NA_INTEGER){
          snprintf(buf, 32, "%.15g", val);
        } else if(signif){
          //use signifant digits rather than decimal digits
          snprintf(buf, 32, "%.*g", sig_digits, val);
        } else if(precision > -1 && precision < 10 && fabs(val) < 2147483647 && fabs(val) > 1e-5) {
          //preferred method: fast with fixed decimal digits
          //does not support large numbers or scientific notation
          modp_dtoa2(val, buf, precision);
        } else {
          //fall back on sprintf (includes scientific notation)
          //funky formula is mostly to convert decimal digits into significant digits
          int decimals = ceil(fmin(17, fmax(1, log10(fabs(val))) + precision));
          snprintf(buf, 32, "%.*g", decimals, val);
        }
        //if always_decimal = TRUE, then append .0 to whole numbers
        if(always_dec && strspn(buf, "0123456789-") == strlen(buf)){
          strcat(buf, ".0");
        }
        SET_STRING_ELT(out, i, mkChar(buf));
      }
    }
  } else {
    error("num_to_char called with invalid object type.");
  }

  UNPROTECT(1);
  return out;
}

SEXP R_parse(SEXP x, SEXP bigint_as_char) {
    /* get data from R */
    const char* json = translateCharUTF8(asChar(x));
    const int bigint = asLogical(bigint_as_char);

    /* ignore BOM as suggested by RFC */
    if(json[0] == '\xEF' && json[1] == '\xBB' && json[2] == '\xBF'){
      warningcall(R_NilValue, "JSON string contains (illegal) UTF8 byte-order-mark!");
      json = json + 3;
    }

    /* ignore rfc7464 record separator */
    if(json[0] == '\x1E'){
      json = json + 1;
    }

    /* parse json */
    char errbuf[1024];
    yajl_val node = yajl_tree_parse(json, errbuf, sizeof(errbuf));

    /* parser error */
    if (!node) {
      Rf_errorcall(R_NilValue, "%s", errbuf);
    }
    SEXP out = ParseValue(node, bigint);
    yajl_tree_free(node);
    return(out);
}

SEXP R_validate(SEXP x) {
    /* get data from R */
    const char* json = translateCharUTF8(asChar(x));

    /* test for BOM */
    if(json[0] == '\xEF' && json[1] == '\xBB' && json[2] == '\xBF'){
      SEXP output = PROTECT(duplicate(ScalarLogical(0)));
      SEXP msg = PROTECT(Rf_mkString("JSON string contains UTF8 byte-order-mark."));
      setAttrib(output, install("err"), msg);
      UNPROTECT(2);
      return(output);
    }

    /* allocate a parser */
    yajl_handle hand = yajl_alloc(NULL, NULL, NULL);

    /* parser options */
    //yajl_config(hand, yajl_dont_validate_strings, 1);

    /* go parse */
    const size_t rd = strlen(json);
    yajl_status stat = yajl_parse(hand, (const unsigned char*) json, rd);
    if(stat == yajl_status_ok) {
      stat = yajl_complete_parse(hand);
    }

    SEXP output = PROTECT(duplicate(ScalarLogical(!stat)));

    //error message
    if (stat != yajl_status_ok) {
        unsigned char* str = yajl_get_error(hand, 1, (const unsigned char*) json, rd);
        SEXP errstr = PROTECT(mkString((const char *) str));
        SEXP offset = PROTECT(ScalarInteger(yajl_get_bytes_consumed(hand)));
        yajl_free_error(hand, str);
        setAttrib(output, install("offset"), offset);
        setAttrib(output, install("err"), errstr);
        UNPROTECT(2);
    }

    /* return boolean vec (0 means no errors, means is valid) */
    yajl_free(hand);
    UNPROTECT(1);
    return output;
}

static void append_text(char **cur, const char* val, int n){
  if(n < 0)
    n = strlen(val);
  memcpy(*cur, val, n);
  *cur += n;
}

static void append_whitespace(char** cur, size_t n, char indent_char){
  memset(*cur, indent_char, n);
  *cur += n;
}

static int array_add_value (context_t *ctx,
                            yajl_val array, yajl_val value)
{
    yajl_val *tmp;

    /* We're checking for NULL pointers in "context_add_value" or its
     * callers. */
    assert (ctx != NULL);
    assert (array != NULL);
    assert (value != NULL);

    /* "context_add_value" will only call us with array values. */
    assert(YAJL_IS_ARRAY(array));

    tmp = realloc(array->u.array.values,
                  sizeof(*(array->u.array.values)) * (array->u.array.len + 1));
    if (tmp == NULL)
        RETURN_ERROR(ctx, ENOMEM, "Out of memory");
    array->u.array.values = tmp;
    array->u.array.values[array->u.array.len] = value;
    array->u.array.len++;

    return 0;
}

unsigned char * base64_decode(const unsigned char *src, size_t len,
			      size_t *out_len)
{
	unsigned char dtable[256], *out, *pos, in[4], block[4], tmp;
	size_t i, count;

	memset(dtable, 0x80, 256);
	for (i = 0; i < sizeof(base64_table); i++)
		dtable[base64_table[i]] = i;
	dtable['='] = 0;

	count = 0;
	for (i = 0; i < len; i++) {
		if (dtable[src[i]] != 0x80)
			count++;
	}

	if (count % 4)
		return NULL;

	pos = out = malloc(count);
	if (out == NULL)
		return NULL;

	count = 0;
	for (i = 0; i < len; i++) {
		tmp = dtable[src[i]];
		if (tmp == 0x80)
			continue;

		in[count] = src[i];
		block[count] = tmp;
		count++;
		if (count == 4) {
			*pos++ = (block[0] << 2) | (block[1] >> 4);
			*pos++ = (block[1] << 4) | (block[2] >> 2);
			*pos++ = (block[2] << 6) | block[3];
			count = 0;
		}
	}

	if (pos > out) {
		if (in[2] == '=')
			pos -= 2;
		else if (in[3] == '=')
			pos--;
	}

	*out_len = pos - out;
	return out;
}

static yajl_val context_pop(context_t *ctx)
{
    stack_elem_t *stack;
    yajl_val v;

    if (ctx->stack == NULL)
        RETURN_ERROR (ctx, NULL, "context_pop: "
                      "Bottom of stack reached prematurely");

    stack = ctx->stack;
    ctx->stack = stack->next;

    v = stack->value;

    free (stack->key);
    free (stack);

    return (v);
}

static int handle_end_array (void *ctx)
{
    yajl_val v;

    v = context_pop (ctx);
    if (v == NULL)
        return (STATUS_ABORT);

    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static int handle_end_map (void *ctx)
{
    yajl_val v;

    v = context_pop (ctx);
    if (v == NULL)
        return (STATUS_ABORT);

    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static bool is_namedlist(SEXP x) {
  if(TYPEOF(x) == VECSXP && getAttrib(x, R_NamesSymbol) != R_NilValue){
    return true;
  }
  return false;
}

static bool is_namedlist_or_null(SEXP x){
  return (is_namedlist(x) || (x == R_NilValue));
}

static bool is_recordlist(SEXP x){
  bool at_least_one_object = false;
  if(!is_unnamedlist(x)){
    return false;
  }
  int len = length(x);
  if(len < 1){
    return false;
  }
  for (int i=0; i<len; i++) {
    if(!is_namedlist_or_null(VECTOR_ELT(x, i))) return false;
    if(!at_least_one_object && is_namedlist(VECTOR_ELT(x, i))) {
      at_least_one_object = true;
    }
  }
  return at_least_one_object;
}

static bool is_unnamedlist(SEXP x) {
  if(TYPEOF(x) == VECSXP && getAttrib(x, R_NamesSymbol) == R_NilValue){
    return true;
  }
  return false;
}

static int object_add_keyval(context_t *ctx,
                             yajl_val obj, char *key, yajl_val value)
{
    const char **tmpk;
    yajl_val *tmpv;

    /* We're checking for NULL in "context_add_value" or its callers. */
    assert (ctx != NULL);
    assert (obj != NULL);
    assert (key != NULL);
    assert (value != NULL);

    /* We're assuring that "obj" is an object in "context_add_value". */
    assert(YAJL_IS_OBJECT(obj));

    tmpk = realloc((void *) obj->u.object.keys, sizeof(*(obj->u.object.keys)) * (obj->u.object.len + 1));
    if (tmpk == NULL)
        RETURN_ERROR(ctx, ENOMEM, "Out of memory");
    obj->u.object.keys = tmpk;

    tmpv = realloc(obj->u.object.values, sizeof (*obj->u.object.values) * (obj->u.object.len + 1));
    if (tmpv == NULL)
        RETURN_ERROR(ctx, ENOMEM, "Out of memory");
    obj->u.object.values = tmpv;

    obj->u.object.keys[obj->u.object.len] = key;
    obj->u.object.values[obj->u.object.len] = value;
    obj->u.object.len++;

    return (0);
}

yajl_val push_parser_get(yajl_handle handle){
  context_t *ctx = (context_t*) handle->ctx;
  return ctx->root;
}

static void strreverse(char* begin, char* end)
{
	char aux;
	while (end > begin)
		aux = *end, *end-- = *begin, *begin++ = aux;
}

static void yajl_array_free (yajl_val v)
{
    size_t i;

    if (!YAJL_IS_ARRAY(v)) return;

    for (i = 0; i < v->u.array.len; i++)
    {
        yajl_tree_free (v->u.array.values[i]);
        v->u.array.values[i] = NULL;
    }

    free(v->u.array.values);
    free(v);
}

void yajl_buf_append(yajl_buf buf, const void * data, size_t len)
{
    yajl_buf_ensure_available(buf, len);
    if (len > 0) {
        assert(data != NULL);
        memcpy(buf->data + buf->used, data, len);
        buf->used += len;
        buf->data[buf->used] = 0;
    }
}

void yajl_buf_clear(yajl_buf buf)
{
    buf->used = 0;
    if (buf->data) buf->data[buf->used] = 0;
}

const unsigned char * yajl_buf_data(yajl_buf buf)
{
    return buf->data;
}

void yajl_buf_ensure_available(yajl_buf buf, size_t want)
{
    size_t need;
    
    assert(buf != NULL);

    /* first call */
    if (buf->data == NULL) {
        buf->len = YAJL_BUF_INIT_SIZE;
        buf->data = (unsigned char *) YA_MALLOC(buf->alloc, buf->len);
        buf->data[0] = 0;
    }

    need = buf->len;

    if (((buf->used > want) ? buf->used : want) > (size_t)(buf->used + want)) {
        /* We cannot allocate more memory than SIZE_MAX. */
        abort();
    }
    while (want >= (need - buf->used)) {
        if (need >= (size_t)((size_t)(-1)<<1)>>1) {
            /* need would overflow. */
            abort();
        }
        need <<= 1;
    }

    if (need != buf->len) {
        buf->data = (unsigned char *) YA_REALLOC(buf->alloc, buf->data, need);
        buf->len = need;
    }
}

void yajl_buf_free(yajl_buf buf)
{
    assert(buf != NULL);
    if (buf->data) YA_FREE(buf->alloc, buf->data);
    YA_FREE(buf->alloc, buf);
}

size_t yajl_buf_len(yajl_buf buf)
{
    return buf->used;
}

yajl_config(yajl_handle h, yajl_option opt, ...)
{
    int rv = 1;
    va_list ap;
    va_start(ap, opt);

    switch(opt) {
        case yajl_allow_comments:
        case yajl_dont_validate_strings:
        case yajl_allow_trailing_garbage:
        case yajl_allow_multiple_values:
        case yajl_allow_partial_values:
            if (va_arg(ap, int)) h->flags |= opt;
            else h->flags &= ~opt;
            break;
        default:
            rv = 0;
    }
    va_end(ap);

    return rv;
}

yajl_do_finish(yajl_handle hand)
{
    yajl_status stat;
    stat = yajl_do_parse(hand,(const unsigned char *) " ",1);

    if (stat != yajl_status_ok) return stat;

    switch(yajl_bs_current(hand->stateStack))
    {
        case yajl_state_parse_error:
        case yajl_state_lexical_error:
            return yajl_status_error;
        case yajl_state_got_value:
        case yajl_state_parse_complete:
            return yajl_status_ok;
        default:
            if (!(hand->flags & yajl_allow_partial_values))
            {
                yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                hand->parseError = "premature EOF";
                return yajl_status_error;
            }
            return yajl_status_ok;
    }
}

yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
              size_t jsonTextLen)
{
    yajl_tok tok;
    const unsigned char * buf;
    size_t bufLen;
    size_t * offset = &(hand->bytesConsumed);

    *offset = 0;

  around_again:
    switch (yajl_bs_current(hand->stateStack)) {
        case yajl_state_parse_complete:
            if (hand->flags & yajl_allow_multiple_values) {
                yajl_bs_set(hand->stateStack, yajl_state_got_value);
                goto around_again;
            }
            if (!(hand->flags & yajl_allow_trailing_garbage)) {
                if (*offset != jsonTextLen) {
                    tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
                                       offset, &buf, &bufLen);
                    if (tok != yajl_tok_eof) {
                        yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                        hand->parseError = "trailing garbage";
                    }
                    goto around_again;
                }
            }
            return yajl_status_ok;
        case yajl_state_lexical_error:
        case yajl_state_parse_error:
            return yajl_status_error;
        case yajl_state_start:
        case yajl_state_got_value:
        case yajl_state_map_need_val:
        case yajl_state_array_need_val:
        case yajl_state_array_start:  {
            /* for arrays and maps, we advance the state for this
             * depth, then push the state of the next depth.
             * If an error occurs during the parsing of the nesting
             * enitity, the state at this level will not matter.
             * a state that needs pushing will be anything other
             * than state_start */

            yajl_state stateToPush = yajl_state_start;

            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
                               offset, &buf, &bufLen);

            switch (tok) {
                case yajl_tok_eof:
                    return yajl_status_ok;
                case yajl_tok_error:
                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
                    goto around_again;
                case yajl_tok_string:
                    if (hand->callbacks && hand->callbacks->yajl_string) {
                        _CC_CHK(hand->callbacks->yajl_string(hand->ctx,
                                                             buf, bufLen));
                    }
                    break;
                case yajl_tok_string_with_escapes:
                    if (hand->callbacks && hand->callbacks->yajl_string) {
                        yajl_buf_clear(hand->decodeBuf);
                        yajl_string_decode(hand->decodeBuf, buf, bufLen);
                        _CC_CHK(hand->callbacks->yajl_string(
                                    hand->ctx, yajl_buf_data(hand->decodeBuf),
                                    yajl_buf_len(hand->decodeBuf)));
                    }
                    break;
                case yajl_tok_bool:
                    if (hand->callbacks && hand->callbacks->yajl_boolean) {
                        _CC_CHK(hand->callbacks->yajl_boolean(hand->ctx,
                                                              *buf == 't'));
                    }
                    break;
                case yajl_tok_null:
                    if (hand->callbacks && hand->callbacks->yajl_null) {
                        _CC_CHK(hand->callbacks->yajl_null(hand->ctx));
                    }
                    break;
                case yajl_tok_left_bracket:
                    if (hand->callbacks && hand->callbacks->yajl_start_map) {
                        _CC_CHK(hand->callbacks->yajl_start_map(hand->ctx));
                    }
                    stateToPush = yajl_state_map_start;
                    break;
                case yajl_tok_left_brace:
                    if (hand->callbacks && hand->callbacks->yajl_start_array) {
                        _CC_CHK(hand->callbacks->yajl_start_array(hand->ctx));
                    }
                    stateToPush = yajl_state_array_start;
                    break;
                case yajl_tok_integer:
                    if (hand->callbacks) {
                        if (hand->callbacks->yajl_number) {
                            _CC_CHK(hand->callbacks->yajl_number(
                                        hand->ctx,(const char *) buf, bufLen));
                        } else if (hand->callbacks->yajl_integer) {
                            long long int i = 0;
                            errno = 0;
                            i = yajl_parse_integer(buf, bufLen);
                            if ((i == LLONG_MIN || i == LLONG_MAX) &&
                                errno == ERANGE)
                            {
                                yajl_bs_set(hand->stateStack,
                                            yajl_state_parse_error);
                                hand->parseError = "integer overflow" ;
                                /* try to restore error offset */
                                if (*offset >= bufLen) *offset -= bufLen;
                                else *offset = 0;
                                goto around_again;
                            }
                            _CC_CHK(hand->callbacks->yajl_integer(hand->ctx,
                                                                  i));
                        }
                    }
                    break;
                case yajl_tok_double:
                    if (hand->callbacks) {
                        if (hand->callbacks->yajl_number) {
                            _CC_CHK(hand->callbacks->yajl_number(
                                        hand->ctx, (const char *) buf, bufLen));
                        } else if (hand->callbacks->yajl_double) {
                            double d = 0.0;
                            yajl_buf_clear(hand->decodeBuf);
                            yajl_buf_append(hand->decodeBuf, buf, bufLen);
                            buf = yajl_buf_data(hand->decodeBuf);
                            errno = 0;
                            d = strtod((char *) buf, NULL);
                            if ((d == HUGE_VAL || d == -HUGE_VAL) &&
                                errno == ERANGE)
                            {
                                yajl_bs_set(hand->stateStack,
                                            yajl_state_parse_error);
                                hand->parseError = "numeric (floating point) "
                                    "overflow";
                                /* try to restore error offset */
                                if (*offset >= bufLen) *offset -= bufLen;
                                else *offset = 0;
                                goto around_again;
                            }
                            _CC_CHK(hand->callbacks->yajl_double(hand->ctx,
                                                                 d));
                        }
                    }
                    break;
                case yajl_tok_right_brace: {
                    if (yajl_bs_current(hand->stateStack) ==
                        yajl_state_array_start)
                    {
                        if (hand->callbacks &&
                            hand->callbacks->yajl_end_array)
                        {
                            _CC_CHK(hand->callbacks->yajl_end_array(hand->ctx));
                        }
                        yajl_bs_pop(hand->stateStack);
                        goto around_again;
                    }
                    /* intentional fall-through */
                }
                case yajl_tok_colon:
                case yajl_tok_comma:
                case yajl_tok_right_bracket:
                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                    hand->parseError =
                        "unallowed token at this point in JSON text";
                    goto around_again;
                default:
                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                    hand->parseError = "invalid token, internal error";
                    goto around_again;
            }
            /* got a value.  transition depends on the state we're in. */
            {
                yajl_state s = yajl_bs_current(hand->stateStack);
                if (s == yajl_state_start || s == yajl_state_got_value) {
                    yajl_bs_set(hand->stateStack, yajl_state_parse_complete);
                } else if (s == yajl_state_map_need_val) {
                    yajl_bs_set(hand->stateStack, yajl_state_map_got_val);
                } else {
                    yajl_bs_set(hand->stateStack, yajl_state_array_got_val);
                }
            }
            if (stateToPush != yajl_state_start) {
                yajl_bs_push(hand->stateStack, stateToPush);
            }

            goto around_again;
        }
        case yajl_state_map_start:
        case yajl_state_map_need_key: {
            /* only difference between these two states is that in
             * start '}' is valid, whereas in need_key, we've parsed
             * a comma, and a string key _must_ follow */
            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
                               offset, &buf, &bufLen);
            switch (tok) {
                case yajl_tok_eof:
                    return yajl_status_ok;
                case yajl_tok_error:
                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
                    goto around_again;
                case yajl_tok_string_with_escapes:
                    if (hand->callbacks && hand->callbacks->yajl_map_key) {
                        yajl_buf_clear(hand->decodeBuf);
                        yajl_string_decode(hand->decodeBuf, buf, bufLen);
                        buf = yajl_buf_data(hand->decodeBuf);
                        bufLen = yajl_buf_len(hand->decodeBuf);
                    }
                    /* intentional fall-through */
                case yajl_tok_string:
                    if (hand->callbacks && hand->callbacks->yajl_map_key) {
                        _CC_CHK(hand->callbacks->yajl_map_key(hand->ctx, buf,
                                                              bufLen));
                    }
                    yajl_bs_set(hand->stateStack, yajl_state_map_sep);
                    goto around_again;
                case yajl_tok_right_bracket:
                    if (yajl_bs_current(hand->stateStack) ==
                        yajl_state_map_start)
                    {
                        if (hand->callbacks && hand->callbacks->yajl_end_map) {
                            _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx));
                        }
                        yajl_bs_pop(hand->stateStack);
                        goto around_again;
                    }
                default:
                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                    hand->parseError =
                        "invalid object key (must be a string)";
                    goto around_again;
            }
        }
        case yajl_state_map_sep: {
            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
                               offset, &buf, &bufLen);
            switch (tok) {
                case yajl_tok_colon:
                    yajl_bs_set(hand->stateStack, yajl_state_map_need_val);
                    goto around_again;
                case yajl_tok_eof:
                    return yajl_status_ok;
                case yajl_tok_error:
                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
                    goto around_again;
                default:
                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                    hand->parseError = "object key and value must "
                        "be separated by a colon (':')";
                    goto around_again;
            }
        }
        case yajl_state_map_got_val: {
            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
                               offset, &buf, &bufLen);
            switch (tok) {
                case yajl_tok_right_bracket:
                    if (hand->callbacks && hand->callbacks->yajl_end_map) {
                        _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx));
                    }
                    yajl_bs_pop(hand->stateStack);
                    goto around_again;
                case yajl_tok_comma:
                    yajl_bs_set(hand->stateStack, yajl_state_map_need_key);
                    goto around_again;
                case yajl_tok_eof:
                    return yajl_status_ok;
                case yajl_tok_error:
                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
                    goto around_again;
                default:
                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                    hand->parseError = "after key and value, inside map, "
                                       "I expect ',' or '}'";
                    /* try to restore error offset */
                    if (*offset >= bufLen) *offset -= bufLen;
                    else *offset = 0;
                    goto around_again;
            }
        }
        case yajl_state_array_got_val: {
            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
                               offset, &buf, &bufLen);
            switch (tok) {
                case yajl_tok_right_brace:
                    if (hand->callbacks && hand->callbacks->yajl_end_array) {
                        _CC_CHK(hand->callbacks->yajl_end_array(hand->ctx));
                    }
                    yajl_bs_pop(hand->stateStack);
                    goto around_again;
                case yajl_tok_comma:
                    yajl_bs_set(hand->stateStack, yajl_state_array_need_val);
                    goto around_again;
                case yajl_tok_eof:
                    return yajl_status_ok;
                case yajl_tok_error:
                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
                    goto around_again;
                default:
                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
                    hand->parseError =
                        "after array element, I expect ',' or ']'";
                    goto around_again;
            }
        }
    }

    //comment out by jeroen for R CMD check
    //abort();
    return yajl_status_error;
}

yajl_free_error(yajl_handle hand, unsigned char * str)
{
    /* use memory allocation functions if set */
    YA_FREE(&(hand->alloc), str);
}

yajl_gen_alloc(const yajl_alloc_funcs * afs)
{
    yajl_gen g = NULL;
    yajl_alloc_funcs afsBuffer;

    /* first order of business is to set up memory allocation routines */
    if (afs != NULL) {
        if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL)
        {
            return NULL;
        }
    } else {
        yajl_set_default_alloc_funcs(&afsBuffer);
        afs = &afsBuffer;
    }

    g = (yajl_gen) YA_MALLOC(afs, sizeof(struct yajl_gen_t));
    if (!g) return NULL;

    memset((void *) g, 0, sizeof(struct yajl_gen_t));
    /* copy in pointers to allocation routines */
    memcpy((void *) &(g->alloc), (void *) afs, sizeof(yajl_alloc_funcs));

    g->print = &yajl_buf_print;
    g->ctx = yajl_buf_alloc(&(g->alloc));
    g->indentString = "    ";

    return g;
}

yajl_gen_clear(yajl_gen g)
{
    if (g->print == &yajl_buf_print) yajl_buf_clear((yajl_buf)g->ctx);
}

yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...)
{
    int rv = 1;
    va_list ap;
    va_start(ap, opt);

    switch(opt) {
        case yajl_gen_beautify:
        case yajl_gen_validate_utf8:
        case yajl_gen_escape_solidus:
            if (va_arg(ap, int)) g->flags |= opt;
            else g->flags &= ~opt;
            break;
        case yajl_gen_indent_string: {
            const char *indent = va_arg(ap, const char *);
            g->indentString = indent;
            for (; *indent; indent++) {
                if (*indent != '\n'
                    && *indent != '\v'
                    && *indent != '\f'
                    && *indent != '\t'
                    && *indent != '\r'
                    && *indent != ' ')
                {
                    g->indentString = NULL;
                    rv = 0;
                }
            }
            break;
        }
        case yajl_gen_print_callback:
            yajl_buf_free(g->ctx);
            g->print = va_arg(ap, const yajl_print_t);
            g->ctx = va_arg(ap, void *);
            break;
        default:
            rv = 0;
    }

    va_end(ap);

    return rv;
}

yajl_gen_free(yajl_gen g)
{
    if (g->print == &yajl_buf_print) yajl_buf_free((yajl_buf)g->ctx);
    YA_FREE(&(g->alloc), g);
}

yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf,
                 size_t * len)
{
    if (g->print != &yajl_buf_print) return yajl_gen_no_buf;
    *buf = yajl_buf_data((yajl_buf)g->ctx);
    *len = yajl_buf_len((yajl_buf)g->ctx);
    return yajl_gen_status_ok;
}

yajl_get_bytes_consumed(yajl_handle hand)
{
    if (!hand) return 0;
    else return hand->bytesConsumed;
}

yajl_get_error(yajl_handle hand, int verbose,
               const unsigned char * jsonText, size_t jsonTextLen)
{
    return yajl_render_error_string(hand, jsonText, jsonTextLen, verbose);
}

static void yajl_internal_free(void *ctx, void * ptr)
{
    (void)ctx;
    free(ptr);
}

static void * yajl_internal_malloc(void *ctx, size_t sz)
{
    (void)ctx;
    return malloc(sz);
}

static void * yajl_internal_realloc(void *ctx, void * previous,
                                    size_t sz)
{
    (void)ctx;
    return realloc(previous, sz);
}

yajl_lex_alloc(yajl_alloc_funcs * alloc,
               unsigned int allowComments, unsigned int validateUTF8)
{
    yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t));
    memset((void *) lxr, 0, sizeof(struct yajl_lexer_t));
    lxr->buf = yajl_buf_alloc(alloc);
    lxr->allowComments = allowComments;
    lxr->validateUTF8 = validateUTF8;
    lxr->alloc = alloc;
    return lxr;
}

yajl_lex_error_to_string(yajl_lex_error error)
{
    switch (error) {
        case yajl_lex_e_ok:
            return "ok, no error";
        case yajl_lex_string_invalid_utf8:
            return "invalid bytes in UTF8 string.";
        case yajl_lex_string_invalid_escaped_char:
            return "inside a string, '\\' occurs before a character "
                   "which it may not.";
        case yajl_lex_string_invalid_json_char:
            return "invalid character inside string.";
        case yajl_lex_string_invalid_hex_char:
            return "invalid (non-hex) character occurs after '\\u' inside "
                   "string.";
        case yajl_lex_invalid_char:
            return "invalid char in json text.";
        case yajl_lex_invalid_string:
            return "invalid string in json text.";
        case yajl_lex_missing_integer_after_exponent:
            return "malformed number, a digit is required after the exponent.";
        case yajl_lex_missing_integer_after_decimal:
            return "malformed number, a digit is required after the "
                   "decimal point.";
        case yajl_lex_missing_integer_after_minus:
            return "malformed number, a digit is required after the "
                   "minus sign.";
        case yajl_lex_unallowed_comment:
            return "probable comment found in input text, comments are "
                   "not enabled.";
    }
    return "unknown error code";
}

yajl_lex_free(yajl_lexer lxr)
{
    yajl_buf_free(lxr->buf);
    YA_FREE(lxr->alloc, lxr);
    return;
}

yajl_lex_get_error(yajl_lexer lexer)
{
    if (lexer == NULL) return (yajl_lex_error) -1;
    return lexer->error;
}

static void yajl_object_free (yajl_val v)
{
    size_t i;

    if (!YAJL_IS_OBJECT(v)) return;

    for (i = 0; i < v->u.object.len; i++)
    {
        free((char *) v->u.object.keys[i]);
        v->u.object.keys[i] = NULL;
        yajl_tree_free (v->u.object.values[i]);
        v->u.object.values[i] = NULL;
    }

    free((void*) v->u.object.keys);
    free(v->u.object.values);
    free(v);
}

yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
                         size_t jsonTextLen, int verbose)
{
    size_t offset = hand->bytesConsumed;
    unsigned char * str;
    const char * errorType = NULL;
    const char * errorText = NULL;
    char text[72];
    const char * arrow = "                     (right here) ------^\n";

    if (yajl_bs_current(hand->stateStack) == yajl_state_parse_error) {
        errorType = "parse";
        errorText = hand->parseError;
    } else if (yajl_bs_current(hand->stateStack) == yajl_state_lexical_error) {
        errorType = "lexical";
        errorText = yajl_lex_error_to_string(yajl_lex_get_error(hand->lexer));
    } else {
        errorType = "unknown";
    }

    {
        size_t memneeded = 0;
        memneeded += strlen(errorType);
        memneeded += strlen(" error");
        if (errorText != NULL) {
            memneeded += strlen(": ");
            memneeded += strlen(errorText);
        }
        str = (unsigned char *) YA_MALLOC(&(hand->alloc), memneeded + 2);
        if (!str) return NULL;
        str[0] = 0;
        strcat((char *) str, errorType);
        strcat((char *) str, " error");
        if (errorText != NULL) {
            strcat((char *) str, ": ");
            strcat((char *) str, errorText);
        }
        strcat((char *) str, "\n");
    }

    /* now we append as many spaces as needed to make sure the error
     * falls at char 41, if verbose was specified */
    if (verbose) {
        size_t start, end, i;
        size_t spacesNeeded;

        spacesNeeded = (offset < 30 ? 40 - offset : 10);
        start = (offset >= 30 ? offset - 30 : 0);
        end = (offset + 30 > jsonTextLen ? jsonTextLen : offset + 30);

        for (i=0;i<spacesNeeded;i++) text[i] = ' ';

        for (;start < end;start++, i++) {
            if (jsonText[start] != '\n' && jsonText[start] != '\r')
            {
                text[i] = jsonText[start];
            }
            else
            {
                text[i] = ' ';
            }
        }
        assert(i <= 71);
        text[i++] = '\n';
        text[i] = 0;
        {
            char * newStr = (char *)
                YA_MALLOC(&(hand->alloc), (unsigned int)(strlen((char *) str) +
                                                         strlen((char *) text) +
                                                         strlen(arrow) + 1));
            if (newStr) {
                newStr[0] = 0;
                strcat((char *) newStr, (char *) str);
                strcat((char *) newStr, text);
                strcat((char *) newStr, arrow);
            }
            YA_FREE(&(hand->alloc), str);
            str = (unsigned char *) newStr;
        }
    }
    return str;
}

void yajl_set_default_alloc_funcs(yajl_alloc_funcs * yaf)
{
    yaf->malloc = yajl_internal_malloc;
    yaf->free = yajl_internal_free;
    yaf->realloc = yajl_internal_realloc;
    yaf->ctx = NULL;
}

void yajl_tree_free (yajl_val v)
{
    if (v == NULL) return;

    if (YAJL_IS_STRING(v))
    {
        free(v->u.string);
        free(v);
    }
    else if (YAJL_IS_NUMBER(v))
    {
        free(v->u.number.r);
        free(v);
    }
    else if (YAJL_GET_OBJECT(v))
    {
        yajl_object_free(v);
    }
    else if (YAJL_GET_ARRAY(v))
    {
        yajl_array_free(v);
    }
    else /* if (yajl_t_true or yajl_t_false or yajl_t_null) */
    {
        free(v);
    }
}

SEXP C_row_collapse_array(SEXP m, SEXP indent){
  //get matrix dimensions
  int *dims = INTEGER(getAttrib(m, R_DimSymbol));
  int x = dims[0];
  int y = dims[1];

  //allocate the output vector
  SEXP out = PROTECT(allocVector(STRSXP, x));
  SEXP vec = PROTECT(allocVector(STRSXP, y));
  for(int i = 0; i < x; i++) {
    for(int j = 0; j < y; j++) {
      SET_STRING_ELT(vec, j, STRING_ELT(m, j*x + i));
    }
    if(asInteger(indent) == NA_INTEGER){
      SET_STRING_ELT(out, i, STRING_ELT(C_collapse_array(vec), 0));
    } else {
      SET_STRING_ELT(out, i, STRING_ELT(C_collapse_array_pretty_inner(vec), 0));
    }
  }
  UNPROTECT(2);
  return out;
}

SEXP C_row_collapse_object(SEXP names, SEXP m, SEXP indent){
  //get matrix dimensions
  int *dims = INTEGER(getAttrib(m, R_DimSymbol));
  int x = dims[0];
  int y = dims[1];

  //allocate the output vector
  SEXP out = PROTECT(allocVector(STRSXP, x));
  SEXP vec = PROTECT(allocVector(STRSXP, y));
  for(int i = 0; i < x; i++) {
    for(int j = 0; j < y; j++) {
      SET_STRING_ELT(vec, j, STRING_ELT(m, j*x + i));
    }
    if(asInteger(indent) == NA_INTEGER){
      SET_STRING_ELT(out, i, STRING_ELT(C_collapse_object(names, vec), 0));
    } else {
      SET_STRING_ELT(out, i, STRING_ELT(C_collapse_object_pretty(names, vec, indent), 0));
    }
  }
  UNPROTECT(2);
  return out;
}

SEXP C_transpose_list(SEXP x, SEXP names) {
  size_t ncol = Rf_length(names);
  size_t nrow = Rf_length(x);
  SEXP out = PROTECT(allocVector(VECSXP, ncol));
  for(size_t i = 0; i < ncol; i++){
    const char * targetname = CHAR(STRING_ELT(names, i));
    SEXP col = PROTECT(allocVector(VECSXP, nrow));
    for(size_t j = 0; j < nrow; j++){
      //search for 'targetname' in each record j
      SEXP list = VECTOR_ELT(x, j);
      SEXP listnames = getAttrib(list, R_NamesSymbol);
      for(size_t k = 0; k < Rf_length(listnames); k++){
        if(!strcmp(CHAR(STRING_ELT(listnames, k)), targetname)){
          SET_VECTOR_ELT(col, j, VECTOR_ELT(list, k));
          break;
        }
      }
    }
    SET_VECTOR_ELT(out, i, col);
    UNPROTECT(1);
  }
  //setAttrib(out, R_NamesSymbol, names);
  UNPROTECT(1);
  return out;
}

SEXP R_base64_decode(SEXP buf){
  if(TYPEOF(buf) != RAWSXP)
    Rf_error("base64 buf must be raw");
  size_t len = Rf_length(buf);
  size_t outlen = 0;
  unsigned char * out = base64_decode(RAW(buf), len, &outlen);
  if(out == NULL)
    Rf_error("Error in base64 decode");
  SEXP res = allocVector(RAWSXP, outlen);
  memcpy(RAW(res), out, outlen);
  free(out);
  return res;
}

SEXP R_base64_encode(SEXP buf){
  if(TYPEOF(buf) != RAWSXP)
    Rf_error("base64 buf must be raw");
  size_t len = Rf_length(buf);
  size_t outlen = 0;
  unsigned char * out = base64_encode(RAW(buf), len, &outlen);
  if(out == NULL)
    Rf_error("Error in base64 encode");
  SEXP res = PROTECT(allocVector(STRSXP, 1));
  SET_STRING_ELT(res, 0, mkCharLen((char*) out, outlen));
  free(out);
  UNPROTECT(1);
  return res;
}

SEXP R_parse_connection(SEXP sConn, SEXP bigint_as_char){
  int first = 1;
  char errbuf[bufsize];
  unsigned char * errstr;
  yajl_handle push_parser = push_parser_new();
  SEXP call = PROTECT(Rf_lang4(
    PROTECT(Rf_install("readBin")),
    sConn,
    PROTECT(Rf_allocVector(RAWSXP, 0)),
    PROTECT(Rf_ScalarInteger(bufsize))));
  while(1){
    SEXP out = PROTECT(Rf_eval(call, R_BaseEnv));
    int len = Rf_length(out);
    if(len <= 0){
      UNPROTECT(1);
      break;
    }
    unsigned char * ptr = RAW(out);

    //strip off BOM
    if(first && len > 3 && ptr[0] == 239 && ptr[1] == 187 && ptr[2] == 191){
      warningcall(R_NilValue, "JSON string contains (illegal) UTF8 byte-order-mark!");
      ptr += 3;
      len -= 3;
    }

    //strip off rfc7464 record separator
    if(first && len > 1 && ptr[0] == 30){
      ptr += 1;
      len -= 1;
    }

    first = 0;

    /* parse and check for errors */
    if (yajl_parse(push_parser, ptr, len) != yajl_status_ok){
      errstr = yajl_get_error(push_parser, 1, ptr, len);
      goto JSON_FAIL;
    }
    UNPROTECT(1);
  }
  UNPROTECT(4);

  /* complete parse */
  if (yajl_complete_parse(push_parser) != yajl_status_ok){
    errstr = yajl_get_error(push_parser, 1, NULL, 0);
    goto JSON_FAIL;
  }

  /* get output */
  yajl_val tree = push_parser_get(push_parser);
  SEXP out = PROTECT(ParseValue(tree, asLogical(bigint_as_char)));
  yajl_tree_free(tree);
  yajl_free(push_parser);
  UNPROTECT(1);
  return out;

  JSON_FAIL:
    strncpy(errbuf, (char *) errstr, bufsize - 1);
    yajl_free_error(push_parser, errstr);
    yajl_free(push_parser);
    Rf_error("%s", errbuf);
}

SEXP R_reformat(SEXP x, SEXP pretty, SEXP indent_string) {
    yajl_status stat;
    yajl_handle hand;
    yajl_gen g;
    SEXP output;

    /* init generator */
    g = yajl_gen_alloc(NULL);
    yajl_gen_config(g, yajl_gen_beautify, asInteger(pretty));
    yajl_gen_config(g, yajl_gen_indent_string, translateCharUTF8(asChar(indent_string)));
    yajl_gen_config(g, yajl_gen_validate_utf8, 0);
    yajl_gen_config(g, yajl_gen_escape_solidus, 1); //modified to only escape for "</"

    /* init parser */
    hand = yajl_alloc(&callbacks, NULL, (void *) g);
    yajl_config(hand, yajl_allow_comments, 1);

    /* get data from R */
    const char* json = translateCharUTF8(asChar(x));

    /* ignore BOM */
    if(json[0] == '\xEF' && json[1] == '\xBB' && json[2] == '\xBF'){
      json = json + 3;
    }

    /* Get length (after removing bom) */
    const size_t rd = strlen(json);

    /* parse */
    stat = yajl_parse(hand, (const unsigned char*) json, rd);
    if(stat == yajl_status_ok) {
      stat = yajl_complete_parse(hand);
    }

    //error message
    if (stat != yajl_status_ok) {
      unsigned char* str = yajl_get_error(hand, 1, (const unsigned char*) json, rd);
      output = PROTECT(mkString((const char*) str));
      yajl_free_error(hand, str);
    } else {
      //create R object
      const unsigned char* buf;
      size_t len;
      yajl_gen_get_buf(g, &buf, &len);

      //force as UTF8 string
      output = PROTECT(allocVector(STRSXP, 1));
      SET_STRING_ELT(output, 0, mkCharCE((const char*) buf, CE_UTF8));
      setAttrib(output, R_ClassSymbol, mkString("json"));
    }

    /* clean up */
    yajl_gen_clear(g);
    yajl_gen_free(g);
    yajl_free(hand);

    /* return boolean vec (0 means no errors, means is valid) */
    SEXP vec = PROTECT(allocVector(VECSXP, 2));
    SET_VECTOR_ELT(vec, 0, ScalarInteger(stat));
    SET_VECTOR_ELT(vec, 1, output);
    UNPROTECT(2);
    return vec;
}

unsigned char * base64_encode(const unsigned char *src, size_t len,
			      size_t *out_len)
{
	unsigned char *out, *pos;
	const unsigned char *end, *in;
	size_t olen;
	int line_len;

	olen = len * 4 / 3 + 4; /* 3-byte blocks to 4-byte */
	olen += olen / 72; /* line feeds */
	olen++; /* nul termination */
	out = malloc(olen);
	if (out == NULL)
		return NULL;

	end = src + len;
	in = src;
	pos = out;
	line_len = 0;
	while (end - in >= 3) {
		*pos++ = base64_table[in[0] >> 2];
		*pos++ = base64_table[((in[0] & 0x03) << 4) | (in[1] >> 4)];
		*pos++ = base64_table[((in[1] & 0x0f) << 2) | (in[2] >> 6)];
		*pos++ = base64_table[in[2] & 0x3f];
		in += 3;
		line_len += 4;
		if (line_len >= 72) {
			*pos++ = '\n';
			line_len = 0;
		}
	}

	if (end - in) {
		*pos++ = base64_table[in[0] >> 2];
		if (end - in == 1) {
			*pos++ = base64_table[(in[0] & 0x03) << 4];
			*pos++ = '=';
		} else {
			*pos++ = base64_table[((in[0] & 0x03) << 4) |
					      (in[1] >> 4)];
			*pos++ = base64_table[(in[1] & 0x0f) << 2];
		}
		*pos++ = '=';
		line_len += 4;
	}

	//if (line_len)
	//	*pos++ = '\n';

	*pos = '\0';
	if (out_len)
		*out_len = pos - out;
	return out;
}

static int context_push(context_t *ctx, yajl_val v)
{
    stack_elem_t *stack;

    stack = malloc (sizeof (*stack));
    if (stack == NULL)
        RETURN_ERROR (ctx, ENOMEM, "Out of memory");
    memset (stack, 0, sizeof (*stack));

    assert ((ctx->stack == NULL)
            || YAJL_IS_OBJECT (v)
            || YAJL_IS_ARRAY (v));

    stack->value = v;
    stack->next = ctx->stack;
    ctx->stack = stack;

    return (0);
}

static int handle_boolean (void *ctx, int boolean_value)
{
    yajl_val v;

    v = value_alloc (boolean_value ? yajl_t_true : yajl_t_false);
    if (v == NULL)
        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");

    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static int handle_null (void *ctx)
{
    yajl_val v;

    v = value_alloc (yajl_t_null);
    if (v == NULL)
        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");

    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static int handle_number (void *ctx, const char *string, size_t string_length)
{
    yajl_val v;
    char *endptr;

    v = value_alloc(yajl_t_number);
    if (v == NULL)
        RETURN_ERROR((context_t *) ctx, STATUS_ABORT, "Out of memory");

    v->u.number.r = malloc(string_length + 1);
    if (v->u.number.r == NULL)
    {
        free(v);
        RETURN_ERROR((context_t *) ctx, STATUS_ABORT, "Out of memory");
    }
    memcpy(v->u.number.r, string, string_length);
    v->u.number.r[string_length] = 0;

    v->u.number.flags = 0;

    errno = 0;
    v->u.number.i = yajl_parse_integer((const unsigned char *) v->u.number.r,
                                       strlen(v->u.number.r));
    if (errno == 0)
        v->u.number.flags |= YAJL_NUMBER_INT_VALID;

    endptr = NULL;
    errno = 0;
    v->u.number.d = strtod(v->u.number.r, &endptr);
    if ((errno == 0) && (endptr != NULL) && (*endptr == 0))
        v->u.number.flags |= YAJL_NUMBER_DOUBLE_VALID;

    return ((context_add_value(ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static int handle_start_array (void *ctx)
{
    yajl_val v;

    v = value_alloc(yajl_t_array);
    if (v == NULL)
        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");

    v->u.array.values = NULL;
    v->u.array.len = 0;

    return ((context_push (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static int handle_start_map (void *ctx)
{
    yajl_val v;

    v = value_alloc(yajl_t_object);
    if (v == NULL)
        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");

    v->u.object.keys = NULL;
    v->u.object.values = NULL;
    v->u.object.len = 0;

    return ((context_push (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

static int handle_string (void *ctx,
                          const unsigned char *string, size_t string_length)
{
    yajl_val v;

    v = value_alloc (yajl_t_string);
    if (v == NULL)
        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");

    v->u.string = malloc (string_length + 1);
    if (v->u.string == NULL)
    {
        free (v);
        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
    }
    memcpy(v->u.string, string, string_length);
    v->u.string[string_length] = 0;

    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
}

size_t modp_dtoa2(double value, char* str, int prec)
{
	/* Hacky test for NaN
	 * under -fast-math this won't work, but then you also won't
	 * have correct nan values anyways.  The alternative is
	 * to link with libmath (bad) or hack IEEE double bits (bad)
	 */
	if (! (value == value)) {
		str[0] = 'n'; str[1] = 'a'; str[2] = 'n'; str[3] = '\0';
		return (size_t) 3;
	}

	/* if input is larger than thres_max, revert to exponential */
	const double thres_max = (double)(0x7FFFFFFF);

	double diff = 0.0;
	char* wstr = str;

	if (prec < 0) {
		prec = 0;
	} else if (prec > 9) {
		/* precision of >= 10 can lead to overflow errors */
		prec = 9;
	}

	/* we'll work in positive values and deal with the
	   negative sign issue later */
	int neg = 0;
	if (value < 0) {
		neg = 1;
		value = -value;
	}

	int whole = (int) value;
	double tmp = (value - whole) * powers_of_10[prec];
	uint32_t frac = (uint32_t)(tmp);
	diff = tmp - frac;

	if (diff > 0.5) {
		++frac;
		/* handle rollover, e.g.  case 0.99 with prec 1 is 1.0  */
		if (frac >= powers_of_10[prec]) {
			frac = 0;
			++whole;
		}
	} else if (diff == 0.5 && prec > 0 && (frac & 1)) {
		/* if halfway, round up if odd, OR
		   if last digit is 0.  That last part is strange */
		++frac;
		if (frac >= powers_of_10[prec]) {
			frac = 0;
			++whole;
		}
	} else if (diff == 0.5 && prec == 0 && (whole & 1)) {
		++frac;
		if (frac >= powers_of_10[prec]) {
			frac = 0;
			++whole;
		}
	}

	/* for very large numbers switch back to native sprintf for exponentials.
	   anyone want to write code to replace this? */
	/*
	   normal printf behavior is to print EVERY whole number digit
	   which can be 100s of characters overflowing your buffers == bad
	   */
	if (value > thres_max) {
	  snprintf(str, 13, "%e", neg ? -value : value);
		return strlen(str);
	}

	int has_decimal = 0;
	int count = prec;

	/* Remove ending zeros */
	if (prec > 0) {
		while (count > 0 && ((frac % 10) == 0)) {
			count--;
			frac /= 10;
		}
	}

	while (count > 0) {
		--count;
		*wstr++ = (char)(48 + (frac % 10));
		frac /= 10;
		has_decimal = 1;
	}

	if (frac > 0) {
		++whole;
	}

	/* add decimal */
	if (has_decimal) {
		*wstr++ = '.';
	}
	/* do whole part
	 * Take care of sign conversion
	 * Number is reversed.
	 */
	do *wstr++ = (char)(48 + (whole % 10)); while (whole /= 10);
	if (neg) {
		*wstr++ = '-';
	}
	*wstr='\0';
	strreverse(str, wstr-1);
	return (size_t)(wstr - str);
}

size_t modp_itoa10(int32_t value, char* str)
{
	char* wstr=str;
	/* Take care of sign */
	uint32_t uvalue = (value < 0) ? (uint32_t)(-value) : (uint32_t)(value);
	/* Conversion. Number is reversed. */
	do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);
	if (value < 0) *wstr++ = '-';
	*wstr='\0';

	/* Reverse string */
	strreverse(str,wstr-1);
	return (size_t)(wstr - str);
}

size_t modp_litoa10(int64_t value, char* str)
{
	char* wstr=str;
	uint64_t uvalue = (value < 0) ? (uint64_t)(-value) : (uint64_t)(value);

	/* Conversion. Number is reversed. */
	do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);
	if (value < 0) *wstr++ = '-';
	*wstr='\0';

	/* Reverse string */
	strreverse(str,wstr-1);
	return (size_t)(wstr - str);
}

yajl_handle push_parser_new (void) {

  /* init callback handlers */
  yajl_callbacks *callbacks = &mem_callbacks;
  memset(callbacks, 0, sizeof(yajl_callbacks));

  callbacks->yajl_null = handle_null;
  callbacks->yajl_boolean = handle_boolean;
  callbacks->yajl_number = handle_number;
  callbacks->yajl_integer = NULL;
  callbacks->yajl_double = NULL;
  callbacks->yajl_string = handle_string;
  callbacks->yajl_start_map = handle_start_map;
  callbacks->yajl_map_key = handle_string;
  callbacks->yajl_end_map = handle_end_map;
  callbacks->yajl_start_array = handle_start_array;
  callbacks->yajl_end_array = handle_end_array;

  /* init context */
  context_t *ctx = &mem_ctx;
  memset(ctx, 0, sizeof(context_t));

  /* init handle */
  yajl_handle handle = yajl_alloc(callbacks, NULL, ctx);
  yajl_config(handle, yajl_allow_comments, 1);
  return handle;
}

static yajl_val value_alloc (yajl_type type)
{
    yajl_val v;

    v = malloc (sizeof (*v));
    if (v == NULL) return (NULL);
    memset (v, 0, sizeof (*v));
    v->type = type;

    return (v);
}

yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc)
{
    yajl_buf b = YA_MALLOC(alloc, sizeof(struct yajl_buf_t));
    memset((void *) b, 0, sizeof(struct yajl_buf_t));
    b->alloc = alloc;
    return b;
}

static void yajl_buf_print(void * ctx, const char * str, size_t len)
{
  yajl_buf_append(ctx, str, len);
}

yajl_free(yajl_handle handle)
{
    yajl_bs_free(handle->stateStack);
    yajl_buf_free(handle->decodeBuf);
    if (handle->lexer) {
        yajl_lex_free(handle->lexer);
        handle->lexer = NULL;
    }
    YA_FREE(&(handle->alloc), handle);
}

yajl_parse_integer(const unsigned char *number, unsigned int length)
{
    long long ret  = 0;
    long sign = 1;
    const unsigned char *pos = number;
    if (*pos == '-') { pos++; sign = -1; }
    if (*pos == '+') { pos++; }

    while (pos < number + length) {
        if ( ret > MAX_VALUE_TO_MULTIPLY ) {
            errno = ERANGE;
            return sign == 1 ? LLONG_MAX : LLONG_MIN;
        }
        ret *= 10;
        if (LLONG_MAX - ret < (*pos - '0')) {
            errno = ERANGE;
            return sign == 1 ? LLONG_MAX : LLONG_MIN;
        }
        if (*pos < '0' || *pos > '9') {
            errno = ERANGE;
            return sign == 1 ? LLONG_MAX : LLONG_MIN;
        }
        ret += (*pos++ - '0');
    }

    return sign * ret;
}

yajl_val yajl_tree_parse (const char *input,
                          char *error_buffer, size_t error_buffer_size)
{
    static const yajl_callbacks callbacks =
        {
            /* null        = */ handle_null,
            /* boolean     = */ handle_boolean,
            /* integer     = */ NULL,
            /* double      = */ NULL,
            /* number      = */ handle_number,
            /* string      = */ handle_string,
            /* start map   = */ handle_start_map,
            /* map key     = */ handle_string,
            /* end map     = */ handle_end_map,
            /* start array = */ handle_start_array,
            /* end array   = */ handle_end_array
        };

    yajl_handle handle;
    yajl_status status;
    char * internal_err_str;
	context_t ctx = { NULL, NULL, NULL, 0 };

	ctx.errbuf = error_buffer;
	ctx.errbuf_size = error_buffer_size;

    if (error_buffer != NULL)
        memset (error_buffer, 0, error_buffer_size);

    handle = yajl_alloc (&callbacks, NULL, &ctx);
    yajl_config(handle, yajl_allow_comments, 1);

    status = yajl_parse(handle,
                        (unsigned char *) input,
                        strlen (input));

    //fix by jeroen
    if(status == yajl_status_ok){
      status = yajl_complete_parse (handle);
    }
    //end of fix
    if (status != yajl_status_ok) {
        if (error_buffer != NULL && error_buffer_size > 0) {
               internal_err_str = (char *) yajl_get_error(handle, 1,
                     (const unsigned char *) input,
                     strlen(input));
             snprintf(error_buffer, error_buffer_size, "%s", internal_err_str);
             YA_FREE(&(handle->alloc), internal_err_str);
        }
        while(ctx.stack != NULL) {
             yajl_val v = context_pop(&ctx);
             yajl_tree_free(v);
        }
        yajl_free (handle);
        //If the requested memory is not released in time, it will cause memory leakage
        if(ctx.root)
             yajl_tree_free(ctx.root);
        return NULL;
    }

    yajl_free (handle);
    return (ctx.root);
}

SEXP C_collapse_array_pretty_outer(SEXP x, SEXP indent) {
  if (!isString(x))
    error("x must character vector.");

  int len = length(x);
  int ni = asInteger(indent);
  int spaces = asInteger(Rf_getAttrib(indent, Rf_install("indent_spaces")));
  if(ni == NA_INTEGER)
    error("indent must not be NA");
  if(spaces == NA_INTEGER)
    error("spaces must not be NA");
  char indent_char = ' ';
  if(spaces < 0){
    spaces = -1 * spaces;
    indent_char = '\t';
  }

  //calculate required space
  size_t nchar_total = 0;
  for (int i=0; i<len; i++) {
    nchar_total += strlen(translateCharUTF8(STRING_ELT(x, i)));
  }

  //for indent plus ",\n"
  nchar_total += len * (ni + spaces + 2);

  //outer parentheses plus final indent and linebreak and terminator
  nchar_total += ni + 4;

  //allocate memory and create a cursor
  char *str = malloc(nchar_total);
  char *cursor = str;
  char **cur = &cursor;

  //init object
  append_text(cur, "[", 1);
  const char *start = *cur;

  //copy everything
  for (int i=0; i<len; i++) {
    append_text(cur, "\n", 1);
    append_whitespace(cur, ni + spaces, indent_char);
    append_text(cur, translateCharUTF8(STRING_ELT(x, i)), -1);
    append_text(cur, ",", 1);
  }

  //remove trailing ", "
  if(cursor != start){
    cursor[-1] = '\n';
    append_whitespace(cur, ni, indent_char);
  }

  //finish up
  append_text(cur, "]\0", 2);

  //encode as UTF8 string
  SEXP out = PROTECT(allocVector(STRSXP, 1));
  SET_STRING_ELT(out, 0, mkCharCE(str,  CE_UTF8));
  UNPROTECT(1);
  free(str);
  return out;
}

SEXP C_collapse_object(SEXP x, SEXP y) {
  if (!isString(x) || !isString(y))
    error("x and y must character vectors.");

  int len = length(x);
  if (len != length(y))
    error("x and y must same length.");

  size_t nchar_total = 0;

  for (int i=0; i<len; i++) {
    if(STRING_ELT(y, i) == NA_STRING) continue;
    nchar_total += strlen(translateCharUTF8(STRING_ELT(x, i)));
    nchar_total += strlen(translateCharUTF8(STRING_ELT(y, i)));
    nchar_total += 2;
  }

  char *s = malloc(nchar_total + 3); //if len is 0, we need at least: '{}\0'
  char *olds = s;
  size_t size;

  for (int i=0; i<len; i++) {
    if(STRING_ELT(y, i) == NA_STRING) continue;
    s[0] = ',';
    //add x
    size = strlen(translateCharUTF8(STRING_ELT(x, i)));
    memcpy(++s, translateCharUTF8(STRING_ELT(x, i)), size);
    s += size;

    //add :
    s[0] = ':';

    //add y
    size = strlen(translateCharUTF8(STRING_ELT(y, i)));
    memcpy(++s, translateCharUTF8(STRING_ELT(y, i)), size);
    s += size;
  }
  if(olds == s) s++;
  olds[0] = '{';
  s[0] = '}';
  s[1] = '\0';

  //get character encoding from first element
  SEXP out = PROTECT(allocVector(STRSXP, 1));
  SET_STRING_ELT(out, 0, mkCharCE(olds,  CE_UTF8));
  UNPROTECT(1);
  free(olds);
  return out;
}

SEXP C_collapse_object_pretty(SEXP x, SEXP y, SEXP indent) {
  if (!isString(x) || !isString(y))
    error("x and y must character vectors.");
  int ni = asInteger(indent);
  int spaces = asInteger(Rf_getAttrib(indent, Rf_install("indent_spaces")));
  if(ni == NA_INTEGER)
    error("indent must not be NA");
  if(spaces == NA_INTEGER)
    error("ni_inside must not be NA");
  char indent_char = ' ';
  if(spaces < 0){
    spaces = -1 * spaces;
    indent_char = '\t';
  }

  int len = length(x);
  if (len != length(y))
    error("x and y must have same length.");

  //calculate required space
  size_t nchar_total = 0;
  for (int i=0; i<len; i++) {
    if(STRING_ELT(y, i) == NA_STRING) continue;
    nchar_total += strlen(translateCharUTF8(STRING_ELT(x, i)));
    nchar_total += strlen(translateCharUTF8(STRING_ELT(y, i)));
    nchar_total += ni + spaces + 4; //indent plus plus ": " and ",\n"
  }

  //final indent plus curly braces and linebreak and terminator
  nchar_total += (ni + 2 + 2);

  //allocate memory and create a cursor
  char *str = malloc(nchar_total);
  char *cursor = str;
  char **cur = &cursor;

  //init object
  append_text(cur, "{", 1);
  const char *start = *cur;

  //copy everything
  for (int i=0; i<len; i++) {
    if(STRING_ELT(y, i) == NA_STRING) continue;
    append_text(cur, "\n", 1);
    append_whitespace(cur, ni + spaces, indent_char);
    append_text(cur, translateCharUTF8(STRING_ELT(x, i)), -1);
    append_text(cur, ": ", 2);
    append_text(cur, translateCharUTF8(STRING_ELT(y, i)), -1);
    append_text(cur, ",", 1);
  }

  //finalize object
  if(cursor != start){
    cursor[-1] = '\n';
    append_whitespace(cur, ni, indent_char);
  }
  append_text(cur, "}\0", 2);

  //encode as UTF8 string
  SEXP out = PROTECT(allocVector(STRSXP, 1));
  SET_STRING_ELT(out, 0, mkCharCE(str,  CE_UTF8));
  UNPROTECT(1);
  free(str);
  return out;
}

SEXP C_null_to_na(SEXP x) {
  int len = length(x);
  if(len == 0) return x;

  //null always turns into NA
  bool looks_like_character_vector = false;
  for (int i=0; i<len; i++) {
    if(VECTOR_ELT(x, i) == R_NilValue) {
      SET_VECTOR_ELT(x, i, ScalarLogical(NA_LOGICAL));
    } else if(!looks_like_character_vector && TYPEOF(VECTOR_ELT(x, i)) == STRSXP){
      if((strcmp("NA", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) ||
         (strcmp("NaN", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) ||
         (strcmp("Inf", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) ||
         (strcmp("-Inf", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0)) continue;
      looks_like_character_vector = true;
    }
  }

  // if this is a character vector, do not parse NA strings.
  if(looks_like_character_vector) return(x);

  //parse NA strings
  for (int i=0; i<len; i++) {
    if(TYPEOF(VECTOR_ELT(x, i)) == STRSXP){
      if(strcmp("NA", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) {
        SET_VECTOR_ELT(x, i, ScalarLogical(NA_LOGICAL));
        continue;
      }
      if(strcmp("NaN", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) {
        SET_VECTOR_ELT(x, i, ScalarReal(R_NaN));
        continue;
      }
      if(strcmp("Inf", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) {
        SET_VECTOR_ELT(x, i, ScalarReal(R_PosInf));
        continue;
      }
      if(strcmp("-Inf", CHAR(STRING_ELT(VECTOR_ELT(x, i), 0))) == 0) {
        SET_VECTOR_ELT(x, i, ScalarReal(R_NegInf));
        continue;
      }
    }
  }

  //return updated list
  return x;
}

static int context_add_value (context_t *ctx, yajl_val v)
{
    /* We're checking for NULL values in all the calling functions. */
    assert (ctx != NULL);
    assert (v != NULL);

    /*
     * There are three valid states in which this function may be called:
     *   - There is no value on the stack => This is the only value. This is the
     *     last step done when parsing a document. We assign the value to the
     *     "root" member and return.
     *   - The value on the stack is an object. In this case store the key on the
     *     stack or, if the key has already been read, add key and value to the
     *     object.
     *   - The value on the stack is an array. In this case simply add the value
     *     and return.
     */
    if (ctx->stack == NULL)
    {
        assert (ctx->root == NULL);
        ctx->root = v;
        return (0);
    }
    else if (YAJL_IS_OBJECT (ctx->stack->value))
    {
        if (ctx->stack->key == NULL)
        {
            if (!YAJL_IS_STRING (v))
                RETURN_ERROR (ctx, EINVAL, "context_add_value: "
                              "Object key is not a string (%#04x)",
                              v->type);

            ctx->stack->key = v->u.string;
            v->u.string = NULL;
            free(v);
            return (0);
        }
        else /* if (ctx->key != NULL) */
        {
            char * key;

            key = ctx->stack->key;
            ctx->stack->key = NULL;
            return (object_add_keyval (ctx, ctx->stack->value, key, v));
        }
    }
    else if (YAJL_IS_ARRAY (ctx->stack->value))
    {
        return (array_add_value (ctx, ctx->stack->value, v));
    }
    else
    {
        RETURN_ERROR (ctx, EINVAL, "context_add_value: Cannot add value to "
                      "a value of type %#04x (not a composite type)",
                      ctx->stack->value->type);
    }
}

yajl_alloc(const yajl_callbacks * callbacks,
           yajl_alloc_funcs * afs,
           void * ctx)
{
    yajl_handle hand = NULL;
    yajl_alloc_funcs afsBuffer;

    /* first order of business is to set up memory allocation routines */
    if (afs != NULL) {
        if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL)
        {
            return NULL;
        }
    } else {
        yajl_set_default_alloc_funcs(&afsBuffer);
        afs = &afsBuffer;
    }

    hand = (yajl_handle) YA_MALLOC(afs, sizeof(struct yajl_handle_t));

    /* copy in pointers to allocation routines */
    memcpy((void *) &(hand->alloc), (void *) afs, sizeof(yajl_alloc_funcs));

    hand->callbacks = callbacks;
    hand->ctx = ctx;
    hand->lexer = NULL; 
    hand->bytesConsumed = 0;
    hand->decodeBuf = yajl_buf_alloc(&(hand->alloc));
    hand->flags	    = 0;
    yajl_bs_init(hand->stateStack, &(hand->alloc));
    yajl_bs_push(hand->stateStack, yajl_state_start);

    return hand;
}

yajl_complete_parse(yajl_handle hand)
{
    /* The lexer is lazy allocated in the first call to parse.  if parse is
     * never called, then no data was provided to parse at all.  This is a
     * "premature EOF" error unless yajl_allow_partial_values is specified.
     * allocating the lexer now is the simplest possible way to handle this
     * case while preserving all the other semantics of the parser
     * (multiple values, partial values, etc). */
    if (hand->lexer == NULL) {
        hand->lexer = yajl_lex_alloc(&(hand->alloc),
                                     hand->flags & yajl_allow_comments,
                                     !(hand->flags & yajl_dont_validate_strings));
    }

    return yajl_do_finish(hand);
}

yajl_parse(yajl_handle hand, const unsigned char * jsonText,
           size_t jsonTextLen)
{
    yajl_status status;

    /* lazy allocation of the lexer */
    if (hand->lexer == NULL) {
        hand->lexer = yajl_lex_alloc(&(hand->alloc),
                                     hand->flags & yajl_allow_comments,
                                     !(hand->flags & yajl_dont_validate_strings));
    }

    status = yajl_do_parse(hand, jsonText, jsonTextLen);
    return status;
}