Commit message: Set a limit of parser recursion

Our parser uses recursion, so it causes stack overflow when parsing
deeply nested capture groups. E.g.:

  x2("(" * 32767 + "a" + ")" * 32767, "a", 0, 1)

Set a limit for this.
The default value is defined in regint.h:
* DEFAULT_PARSE_DEPTH_LIMIT (Currently 4096)

Also add two APIs to support this:
* onig_get_parse_depth_limit
* onig_set_parse_depth_limit
CVE: CVE-2017-6181
Vulnerability description: The parse_char_class function in regparse.c in the Onigmo (aka Oniguruma-mod) regular expression library, as used in Ruby 2.4.0, allows remote attackers to cause a denial of service (deep recursion and application crash) via a crafted regular expression.

Project: oniguruma
File: regerror.c

onig_error_code_to_format(int code)
{
  char *p;

  if (code >= 0) return (UChar* )0;

  switch (code) {
  case ONIG_MISMATCH:
    p = "mismatch"; break;
  case ONIG_NO_SUPPORT_CONFIG:
    p = "no support in this configuration"; break;
  case ONIGERR_MEMORY:
    p = "fail to memory allocation"; break;
  case ONIGERR_MATCH_STACK_LIMIT_OVER:
    p = "match-stack limit over"; break;
  case ONIGERR_PARSE_DEPTH_LIMIT_OVER:
    p = "parse depth limit over"; break;
  case ONIGERR_TYPE_BUG:
    p = "undefined type (bug)"; break;
  case ONIGERR_PARSER_BUG:
    p = "internal parser error (bug)"; break;
  case ONIGERR_STACK_BUG:
    p = "stack error (bug)"; break;
  case ONIGERR_UNDEFINED_BYTECODE:
    p = "undefined bytecode (bug)"; break;
  case ONIGERR_UNEXPECTED_BYTECODE:
    p = "unexpected bytecode (bug)"; break;
  case ONIGERR_DEFAULT_ENCODING_IS_NOT_SETTED:
    p = "default multibyte-encoding is not setted"; break;
  case ONIGERR_SPECIFIED_ENCODING_CANT_CONVERT_TO_WIDE_CHAR:
    p = "can't convert to wide-char on specified multibyte-encoding"; break;
  case ONIGERR_FAIL_TO_INITIALIZE:
    p = "fail to initialize"; break;
  case ONIGERR_INVALID_ARGUMENT:
    p = "invalid argument"; break;
  case ONIGERR_END_PATTERN_AT_LEFT_BRACE:
    p = "end pattern at left brace"; break;
  case ONIGERR_END_PATTERN_AT_LEFT_BRACKET:
    p = "end pattern at left bracket"; break;
  case ONIGERR_EMPTY_CHAR_CLASS:
    p = "empty char-class"; break;
  case ONIGERR_PREMATURE_END_OF_CHAR_CLASS:
    p = "premature end of char-class"; break;
  case ONIGERR_END_PATTERN_AT_ESCAPE:
    p = "end pattern at escape"; break;
  case ONIGERR_END_PATTERN_AT_META:
    p = "end pattern at meta"; break;
  case ONIGERR_END_PATTERN_AT_CONTROL:
    p = "end pattern at control"; break;
  case ONIGERR_META_CODE_SYNTAX:
    p = "invalid meta-code syntax"; break;
  case ONIGERR_CONTROL_CODE_SYNTAX:
    p = "invalid control-code syntax"; break;
  case ONIGERR_CHAR_CLASS_VALUE_AT_END_OF_RANGE:
    p = "char-class value at end of range"; break;
  case ONIGERR_CHAR_CLASS_VALUE_AT_START_OF_RANGE:
    p = "char-class value at start of range"; break;
  case ONIGERR_UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS:
    p = "unmatched range specifier in char-class"; break;
  case ONIGERR_TARGET_OF_REPEAT_OPERATOR_NOT_SPECIFIED:
    p = "target of repeat operator is not specified"; break;
  case ONIGERR_TARGET_OF_REPEAT_OPERATOR_INVALID:
    p = "target of repeat operator is invalid"; break;
  case ONIGERR_NESTED_REPEAT_OPERATOR:
    p = "nested repeat operator"; break;
  case ONIGERR_UNMATCHED_CLOSE_PARENTHESIS:
    p = "unmatched close parenthesis"; break;
  case ONIGERR_END_PATTERN_WITH_UNMATCHED_PARENTHESIS:
    p = "end pattern with unmatched parenthesis"; break;
  case ONIGERR_END_PATTERN_IN_GROUP:
    p = "end pattern in group"; break;
  case ONIGERR_UNDEFINED_GROUP_OPTION:
    p = "undefined group option"; break;
  case ONIGERR_INVALID_POSIX_BRACKET_TYPE:
    p = "invalid POSIX bracket type"; break;
  case ONIGERR_INVALID_LOOK_BEHIND_PATTERN:
    p = "invalid pattern in look-behind"; break;
  case ONIGERR_INVALID_REPEAT_RANGE_PATTERN:
    p = "invalid repeat range {lower,upper}"; break;
  case ONIGERR_TOO_BIG_NUMBER:
    p = "too big number"; break;
  case ONIGERR_TOO_BIG_NUMBER_FOR_REPEAT_RANGE:
    p = "too big number for repeat range"; break;
  case ONIGERR_UPPER_SMALLER_THAN_LOWER_IN_REPEAT_RANGE:
    p = "upper is smaller than lower in repeat range"; break;
  case ONIGERR_EMPTY_RANGE_IN_CHAR_CLASS:
    p = "empty range in char class"; break;
  case ONIGERR_MISMATCH_CODE_LENGTH_IN_CLASS_RANGE:
    p = "mismatch multibyte code length in char-class range"; break;
  case ONIGERR_TOO_MANY_MULTI_BYTE_RANGES:
    p = "too many multibyte code ranges are specified"; break;
  case ONIGERR_TOO_SHORT_MULTI_BYTE_STRING:
    p = "too short multibyte code string"; break;
  case ONIGERR_TOO_BIG_BACKREF_NUMBER:
    p = "too big backref number"; break;
  case ONIGERR_INVALID_BACKREF:
#ifdef USE_NAMED_GROUP
    p = "invalid backref number/name"; break;
#else
    p = "invalid backref number"; break;
#endif
  case ONIGERR_NUMBERED_BACKREF_OR_CALL_NOT_ALLOWED:
    p = "numbered backref/call is not allowed. (use name)"; break;
  case ONIGERR_TOO_MANY_CAPTURES:
    p = "too many captures"; break;
  case ONIGERR_TOO_BIG_WIDE_CHAR_VALUE:
    p = "too big wide-char value"; break;
  case ONIGERR_TOO_LONG_WIDE_CHAR_VALUE:
    p = "too long wide-char value"; break;
  case ONIGERR_INVALID_CODE_POINT_VALUE:
    p = "invalid code point value"; break;
  case ONIGERR_EMPTY_GROUP_NAME:
    p = "group name is empty"; break;
  case ONIGERR_INVALID_GROUP_NAME:
    p = "invalid group name <%n>"; break;
  case ONIGERR_INVALID_CHAR_IN_GROUP_NAME:
#ifdef USE_NAMED_GROUP
    p = "invalid char in group name <%n>"; break;
#else
    p = "invalid char in group number <%n>"; break;
#endif
  case ONIGERR_UNDEFINED_NAME_REFERENCE:
    p = "undefined name <%n> reference"; break;
  case ONIGERR_UNDEFINED_GROUP_REFERENCE:
    p = "undefined group <%n> reference"; break;
  case ONIGERR_MULTIPLEX_DEFINED_NAME:
    p = "multiplex defined name <%n>"; break;
  case ONIGERR_MULTIPLEX_DEFINITION_NAME_CALL:
    p = "multiplex definition name <%n> call"; break;
  case ONIGERR_NEVER_ENDING_RECURSION:
    p = "never ending recursion"; break;
  case ONIGERR_GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY:
    p = "group number is too big for capture history"; break;
  case ONIGERR_INVALID_CHAR_PROPERTY_NAME:
    p = "invalid character property name {%n}"; break;
  case ONIGERR_NOT_SUPPORTED_ENCODING_COMBINATION:
    p = "not supported encoding combination"; break;
  case ONIGERR_INVALID_COMBINATION_OF_OPTIONS:
    p = "invalid combination of options"; break;
  case ONIGERR_LIBRARY_IS_NOT_INITIALIZED:
    p = "library is not initialized"; break;

  default:
    p = "undefined error code"; break;
  }

  return (UChar* )p;
}

Project: oniguruma
File: regerror.c

onig_snprintf_with_pattern(buf, bufsize, enc, pat, pat_end, fmt, va_alist)
    UChar buf[];
    int bufsize;
    OnigEncoding enc;
    UChar* pat;
    UChar* pat_end;
    const UChar *fmt;
    va_dcl
#endif
{
  int n, need, len;
  UChar *p, *s, *bp;
  UChar bs[6];
  va_list args;

  va_init_list(args, fmt);
  n = xvsnprintf((char* )buf, bufsize, (const char* )fmt, args);
  va_end(args);

  need = (pat_end - pat) * 4 + 4;

  if (n + need < bufsize) {
    xstrcat((char* )buf, ": /", bufsize);
    s = buf + onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, buf);

    p = pat;
    while (p < pat_end) {
      if (*p == '\\') {
	*s++ = *p++;
	len = enclen(enc, p);
	while (len-- > 0) *s++ = *p++;
      }
      else if (*p == '/') {
	*s++ = (unsigned char )'\\';
	*s++ = *p++;
      }
      else if (ONIGENC_IS_MBC_HEAD(enc, p)) {
        len = enclen(enc, p);
        if (ONIGENC_MBC_MINLEN(enc) == 1) {
          while (len-- > 0) *s++ = *p++;
        }
        else { /* for UTF16 */
          int blen;

          while (len-- > 0) {
	    sprint_byte_with_x((char* )bs, (unsigned int )(*p++));
            blen = onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, bs);
            bp = bs;
            while (blen-- > 0) *s++ = *bp++;
          }
        }
      }
      else if (!ONIGENC_IS_CODE_PRINT(enc, *p) &&
	       !ONIGENC_IS_CODE_SPACE(enc, *p)) {
	sprint_byte_with_x((char* )bs, (unsigned int )(*p++));
	len = onigenc_str_bytelen_null(ONIG_ENCODING_ASCII, bs);
        bp = bs;
	while (len-- > 0) *s++ = *bp++;
      }
      else {
	*s++ = *p++;
      }
    }

    *s++ = '/';
    *s   = '\0';
  }
}

Project: oniguruma
File: regparse.c

scan_env_add_mem_entry(ScanEnv* env)
{
  int i, need, alloc;
  Node** p;

  need = env->num_mem + 1;
  if (need > ONIG_MAX_CAPTURE_NUM)
    return ONIGERR_TOO_MANY_CAPTURES;

  if (need >= SCANENV_MEMNODES_SIZE) {
    if (env->mem_alloc <= need) {
      if (IS_NULL(env->mem_nodes_dynamic)) {
        alloc = INIT_SCANENV_MEMNODES_ALLOC_SIZE;
        p = (Node** )xmalloc(sizeof(Node*) * alloc);
        xmemcpy(p, env->mem_nodes_static,
                sizeof(Node*) * SCANENV_MEMNODES_SIZE);
      }
      else {
        alloc = env->mem_alloc * 2;
        p = (Node** )xrealloc(env->mem_nodes_dynamic, sizeof(Node*) * alloc);
      }
      CHECK_NULL_RETURN_MEMERR(p);

      for (i = env->num_mem + 1; i < alloc; i++)
        p[i] = NULL_NODE;

      env->mem_nodes_dynamic = p;
      env->mem_alloc = alloc;
    }
  }

  env->num_mem++;
  return env->num_mem;
}

Project: oniguruma
File: regparse.c

parse_char_class(Node** np, OnigToken* tok, UChar** src, UChar* end,
		 ScanEnv* env)
{
  int r, neg, len, fetched, and_start;
  OnigCodePoint v, vs;
  UChar *p;
  Node* node;
  CClassNode *cc, *prev_cc;
  CClassNode work_cc;

  enum CCSTATE state;
  enum CCVALTYPE val_type, in_type;
  int val_israw, in_israw;

  env->parse_depth++;
  if (env->parse_depth > ParseDepthLimit)
    return ONIGERR_PARSE_DEPTH_LIMIT_OVER;
  prev_cc = (CClassNode* )NULL;
  *np = NULL_NODE;
  r = fetch_token_in_cc(tok, src, end, env);
  if (r == TK_CHAR && tok->u.c == '^' && tok->escaped == 0) {
    neg = 1;
    r = fetch_token_in_cc(tok, src, end, env);
  }
  else {
    neg = 0;
  }

  if (r < 0) return r;
  if (r == TK_CC_CLOSE) {
    if (! code_exist_check((OnigCodePoint )']',
                           *src, env->pattern_end, 1, env))
      return ONIGERR_EMPTY_CHAR_CLASS;

    CC_ESC_WARN(env, (UChar* )"]");
    r = tok->type = TK_CHAR;  /* allow []...] */
  }

  *np = node = node_new_cclass();
  CHECK_NULL_RETURN_MEMERR(node);
  cc = NCCLASS(node);

  and_start = 0;
  state = CCS_START;
  p = *src;
  while (r != TK_CC_CLOSE) {
    fetched = 0;
    switch (r) {
    case TK_CHAR:
    any_char_in:
      len = ONIGENC_CODE_TO_MBCLEN(env->enc, tok->u.c);
      if (len > 1) {
        in_type = CCV_CODE_POINT;
      }
      else if (len < 0) {
        r = len;
        goto err;
      }
      else {
        /* sb_char: */
        in_type = CCV_SB;
      }
      v = (OnigCodePoint )tok->u.c;
      in_israw = 0;
      goto val_entry2;
      break;

    case TK_RAW_BYTE:
      /* tok->base != 0 : octal or hexadec. */
      if (! ONIGENC_IS_SINGLEBYTE(env->enc) && tok->base != 0) {
        UChar buf[ONIGENC_CODE_TO_MBC_MAXLEN];
        UChar* bufe = buf + ONIGENC_CODE_TO_MBC_MAXLEN;
        UChar* psave = p;
        int i, base = tok->base;

        buf[0] = tok->u.c;
        for (i = 1; i < ONIGENC_MBC_MAXLEN(env->enc); i++) {
          r = fetch_token_in_cc(tok, &p, end, env);
          if (r < 0) goto err;
          if (r != TK_RAW_BYTE || tok->base != base) {
            fetched = 1;
            break;
          }
          buf[i] = tok->u.c;
        }

        if (i < ONIGENC_MBC_MINLEN(env->enc)) {
          r = ONIGERR_TOO_SHORT_MULTI_BYTE_STRING;
          goto err;
        }

        len = enclen(env->enc, buf);
        if (i < len) {
          r = ONIGERR_TOO_SHORT_MULTI_BYTE_STRING;
          goto err;
        }
        else if (i > len) { /* fetch back */
          p = psave;
          for (i = 1; i < len; i++) {
            r = fetch_token_in_cc(tok, &p, end, env);
          }
          fetched = 0;
        }

        if (i == 1) {
          v = (OnigCodePoint )buf[0];
          goto raw_single;
        }
        else {
          v = ONIGENC_MBC_TO_CODE(env->enc, buf, bufe);
          in_type = CCV_CODE_POINT;
        }
      }
      else {
        v = (OnigCodePoint )tok->u.c;
      raw_single:
        in_type = CCV_SB;
      }
      in_israw = 1;
      goto val_entry2;
      break;

    case TK_CODE_POINT:
      v = tok->u.code;
      in_israw = 1;
    val_entry:
      len = ONIGENC_CODE_TO_MBCLEN(env->enc, v);
      if (len < 0) {
        r = len;
        goto err;
      }
      in_type = (len == 1 ? CCV_SB : CCV_CODE_POINT);
    val_entry2:
      r = next_state_val(cc, &vs, v, &val_israw, in_israw, in_type, &val_type,
			 &state, env);
      if (r != 0) goto err;
      break;

    case TK_POSIX_BRACKET_OPEN:
      r = parse_posix_bracket(cc, &p, end, env);
      if (r < 0) goto err;
      if (r == 1) {  /* is not POSIX bracket */
        CC_ESC_WARN(env, (UChar* )"[");
        p = tok->backp;
        v = (OnigCodePoint )tok->u.c;
        in_israw = 0;
        goto val_entry;
      }
      goto next_class;
      break;

    case TK_CHAR_TYPE:
      r = add_ctype_to_cc(cc, tok->u.prop.ctype, tok->u.prop.not, env);
      if (r != 0) return r;

    next_class:
      r = next_state_class(cc, &vs, &val_type, &state, env);
      if (r != 0) goto err;
      break;

    case TK_CHAR_PROPERTY:
      {
        int ctype;

        ctype = fetch_char_property_to_ctype(&p, end, env);
        if (ctype < 0) return ctype;
        r = add_ctype_to_cc(cc, ctype, tok->u.prop.not, env);
        if (r != 0) return r;
        goto next_class;
      }
      break;

    case TK_CC_RANGE:
      if (state == CCS_VALUE) {
        r = fetch_token_in_cc(tok, &p, end, env);
        if (r < 0) goto err;
        fetched = 1;
        if (r == TK_CC_CLOSE) { /* allow [x-] */
        range_end_val:
          v = (OnigCodePoint )'-';
          in_israw = 0;
          goto val_entry;
        }
        else if (r == TK_CC_AND) {
          CC_ESC_WARN(env, (UChar* )"-");
          goto range_end_val;
        }
        state = CCS_RANGE;
      }
      else if (state == CCS_START) {
        /* [-xa] is allowed */
        v = (OnigCodePoint )tok->u.c;
        in_israw = 0;

        r = fetch_token_in_cc(tok, &p, end, env);
        if (r < 0) goto err;
        fetched = 1;
        /* [--x] or [a&&-x] is warned. */
        if (r == TK_CC_RANGE || and_start != 0)
          CC_ESC_WARN(env, (UChar* )"-");

        goto val_entry;
      }
      else if (state == CCS_RANGE) {
        CC_ESC_WARN(env, (UChar* )"-");
        goto any_char_in;  /* [!--x] is allowed */
      }
      else { /* CCS_COMPLETE */
        r = fetch_token_in_cc(tok, &p, end, env);
        if (r < 0) goto err;
        fetched = 1;
        if (r == TK_CC_CLOSE) goto range_end_val; /* allow [a-b-] */
        else if (r == TK_CC_AND) {
          CC_ESC_WARN(env, (UChar* )"-");
          goto range_end_val;
        }
	
        if (IS_SYNTAX_BV(env->syntax, ONIG_SYN_ALLOW_DOUBLE_RANGE_OP_IN_CC)) {
          CC_ESC_WARN(env, (UChar* )"-");
          goto any_char_in;   /* [0-9-a] is allowed as [0-9\-a] */
        }
        r = ONIGERR_UNMATCHED_RANGE_SPECIFIER_IN_CHAR_CLASS;
        goto err;
      }
      break;

    case TK_CC_CC_OPEN: /* [ */
      {
        Node *anode;
        CClassNode* acc;

        r = parse_char_class(&anode, tok, &p, end, env);
        if (r != 0) {
	  onig_node_free(anode);
	  goto cc_open_err;
	}
        acc = NCCLASS(anode);
        r = or_cclass(cc, acc, env->enc);

        onig_node_free(anode);
      cc_open_err:
        if (r != 0) goto err;
      }
      break;

    case TK_CC_AND: /* && */
      {
        if (state == CCS_VALUE) {
          r = next_state_val(cc, &vs, 0, &val_israw, 0, val_type,
                             &val_type, &state, env);
          if (r != 0) goto err;
        }
        /* initialize local variables */
        and_start = 1;
        state = CCS_START;

        if (IS_NOT_NULL(prev_cc)) {
          r = and_cclass(prev_cc, cc, env->enc);
          if (r != 0) goto err;
          bbuf_free(cc->mbuf);
        }
        else {
          prev_cc = cc;
          cc = &work_cc;
        }
        initialize_cclass(cc);
      }
      break;

    case TK_EOT:
      r = ONIGERR_PREMATURE_END_OF_CHAR_CLASS;
      goto err;
      break;
    default:
      r = ONIGERR_PARSER_BUG;
      goto err;
      break;
    }

    if (fetched)
      r = tok->type;
    else {
      r = fetch_token_in_cc(tok, &p, end, env);
      if (r < 0) goto err;
    }
  }

  if (state == CCS_VALUE) {
    r = next_state_val(cc, &vs, 0, &val_israw, 0, val_type,
		       &val_type, &state, env);
    if (r != 0) goto err;
  }

  if (IS_NOT_NULL(prev_cc)) {
    r = and_cclass(prev_cc, cc, env->enc);
    if (r != 0) goto err;
    bbuf_free(cc->mbuf);
    cc = prev_cc;
  }

  if (neg != 0)
    NCCLASS_SET_NOT(cc);
  else
    NCCLASS_CLEAR_NOT(cc);
  if (IS_NCCLASS_NOT(cc) &&
      IS_SYNTAX_BV(env->syntax, ONIG_SYN_NOT_NEWLINE_IN_NEGATIVE_CC)) {
    int is_empty;

    is_empty = (IS_NULL(cc->mbuf) ? 1 : 0);
    if (is_empty != 0)
      BITSET_IS_EMPTY(cc->bs, is_empty);

    if (is_empty == 0) {
#define NEWLINE_CODE    0x0a

      if (ONIGENC_IS_CODE_NEWLINE(env->enc, NEWLINE_CODE)) {
        if (ONIGENC_CODE_TO_MBCLEN(env->enc, NEWLINE_CODE) == 1)
          BITSET_SET_BIT(cc->bs, NEWLINE_CODE);
        else
          add_code_range(&(cc->mbuf), env, NEWLINE_CODE, NEWLINE_CODE);
      }
    }
  }
  *src = p;
  env->parse_depth--;
  return 0;

 err:
  if (cc != NCCLASS(*np))
    bbuf_free(cc->mbuf);
  return r;
}