Program: 45370
Project: 45370
Buggy source:
public int nexttoken(boolean resetState){
  String doctagname="String_Node_Str";
  StringBuilder doctextbuf=null;
  int startofxml=pos();
  StringBuilder blockcommentbuf=null;
  char regexp_flags=0;
  boolean maybe_reserved=false;
  char c=0;
  if (resetState) {
    isFirstTokenOnLine=false;
  }
  while (true) {
    if (debug) {
      System.out.println("String_Node_Str" + state + "String_Node_Str"+ pos());
    }
switch (state) {
case start_state:
{
        c=nextchar();
        mark();
switch (c) {
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
          maybe_reserved=true;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
case '$':
        state=A_state;
      continue;
case 0xffef:
    if (nextchar() == 0xffffffbb && nextchar() == 0xffffffbf) {
      state=start_state;
    }
 else {
      state=error_state;
    }
  continue;
case '@':
return makeToken(ATSIGN_TOKEN);
case '\'':
case '\"':
{
char startquote=(char)c;
boolean needs_escape=false;
while ((c=nextchar()) != startquote) {
  if (c == '\\') {
    needs_escape=true;
    c=nextchar();
    if (c == '\r')     c=nextchar();
    if (c == '\n')     continue;
  }
 else   if (c == '\r' || c == '\n') {
    if (startquote == '\'')     error(kError_Lexical_LineTerminatorInSingleQuotedStringLiteral);
 else     error(kError_Lexical_LineTerminatorInDoubleQuotedStringLiteral);
    break;
  }
 else   if (c == 0) {
    error(kError_Lexical_EndOfStreamInStringLiteral);
    return makeToken(EOS_TOKEN);
  }
}
return makeToken(STRINGLITERAL_TOKEN,input.copyReplaceStringEscapes(needs_escape));
}
case '-':
switch (nextchar()) {
case '-':
return makeToken(MINUSMINUS_TOKEN);
case '=':
return makeToken(MINUSASSIGN_TOKEN);
default :
retract();
return makeToken(MINUS_TOKEN);
}
case '!':
if (nextchar() == '=') {
if (nextchar() == '=') return makeToken(STRICTNOTEQUALS_TOKEN);
retract();
return makeToken(NOTEQUALS_TOKEN);
}
retract();
return makeToken(NOT_TOKEN);
case '%':
switch (nextchar()) {
case '=':
return makeToken(MODULUSASSIGN_TOKEN);
default :
retract();
return makeToken(MODULUS_TOKEN);
}
case '&':
c=nextchar();
if (c == '=') return makeToken(BITWISEANDASSIGN_TOKEN);
if (c == '&') {
if (nextchar() == '=') return makeToken(LOGICALANDASSIGN_TOKEN);
retract();
return makeToken(LOGICALAND_TOKEN);
}
retract();
return makeToken(BITWISEAND_TOKEN);
case '#':
if (HAS_HASHPRAGMAS) {
return makeToken(USE_TOKEN);
}
state=error_state;
continue;
case '(':
return makeToken(LEFTPAREN_TOKEN);
case ')':
return makeToken(RIGHTPAREN_TOKEN);
case '*':
if (nextchar() == '=') return makeToken(MULTASSIGN_TOKEN);
retract();
return makeToken(MULT_TOKEN);
case ',':
return makeToken(COMMA_TOKEN);
case '.':
state=dot_state;
continue;
case '/':
state=slash_state;
continue;
case ':':
if (nextchar() == ':') {
return makeToken(DOUBLECOLON_TOKEN);
}
retract();
return makeToken(COLON_TOKEN);
case ';':
return makeToken(SEMICOLON_TOKEN);
case '?':
return makeToken(QUESTIONMARK_TOKEN);
case '[':
return makeToken(LEFTBRACKET_TOKEN);
case ']':
return makeToken(RIGHTBRACKET_TOKEN);
case '^':
if (nextchar() == '=') return makeToken(BITWISEXORASSIGN_TOKEN);
retract();
return makeToken(BITWISEXOR_TOKEN);
case '{':
return makeToken(LEFTBRACE_TOKEN);
case '|':
c=nextchar();
if (c == '=') return makeToken(BITWISEORASSIGN_TOKEN);
if (c == '|') {
if (nextchar() == '=') return makeToken(LOGICALORASSIGN_TOKEN);
retract();
return makeToken(LOGICALOR_TOKEN);
}
retract();
return makeToken(BITWISEOR_TOKEN);
case '}':
return makeToken(RIGHTBRACE_TOKEN);
case '~':
return makeToken(BITWISENOT_TOKEN);
case '+':
c=nextchar();
if (c == '+') return makeToken(PLUSPLUS_TOKEN);
if (c == '=') return makeToken(PLUSASSIGN_TOKEN);
retract();
return makeToken(PLUS_TOKEN);
case '<':
switch (nextchar()) {
case '<':
if (nextchar() == '=') return makeToken(LEFTSHIFTASSIGN_TOKEN);
retract();
return makeToken(LEFTSHIFT_TOKEN);
case '=':
return makeToken(LESSTHANOREQUALS_TOKEN);
case '/':
return makeToken(XMLTAGSTARTEND_TOKEN);
case '!':
state=xmlcommentorcdatastart_state;
continue;
case '?':
state=xmlpi_state;
continue;
}
retract();
return makeToken(LESSTHAN_TOKEN);
case '=':
if (nextchar() == '=') {
if (nextchar() == '=') return makeToken(STRICTEQUALS_TOKEN);
retract();
return makeToken(EQUALS_TOKEN);
}
retract();
return makeToken(ASSIGN_TOKEN);
case '>':
state=start_state;
switch (nextchar()) {
case '>':
switch (nextchar()) {
case '>':
if (nextchar() == '=') return makeToken(UNSIGNEDRIGHTSHIFTASSIGN_TOKEN);
retract();
return makeToken(UNSIGNEDRIGHTSHIFT_TOKEN);
case '=':
return makeToken(RIGHTSHIFTASSIGN_TOKEN);
default :
retract();
return makeToken(RIGHTSHIFT_TOKEN);
}
case '=':
return makeToken(GREATERTHANOREQUALS_TOKEN);
}
retract();
return makeToken(GREATERTHAN_TOKEN);
case '0':
state=zero_state;
continue;
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimalinteger_state;
continue;
case ' ':
case '\t':
case 0x000b:
case 0x000c:
case 0x0085:
case 0x00a0:
continue;
case '\n':
case '\r':
isFirstTokenOnLine=true;
continue;
case 0:
return makeToken(EOS_TOKEN);
default :
switch (input.nextcharClass((char)c,true)) {
case Lu:
case Ll:
case Lt:
case Lm:
case Lo:
case Nl:
maybe_reserved=false;
state=A_state;
continue;
case Zs:
case Cc:
case Cf:
continue;
case Zl:
case Zp:
isFirstTokenOnLine=true;
continue;
default :
state=error_state;
continue;
}
}
}
case A_state:
{
boolean needs_escape=c == '\\';
while (true) {
c=nextchar();
if (c >= 'a' && c <= 'z') {
continue;
}
if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '$' || c == '_') {
maybe_reserved=false;
continue;
}
if (c <= 0x7f) {
if (c == '\\') {
needs_escape=true;
}
 else {
retract();
break;
}
}
switch (input.nextcharClass(c,false)) {
case Lu:
case Ll:
case Lt:
case Lm:
case Lo:
case Nl:
case Mn:
case Mc:
case Nd:
case Pc:
maybe_reserved=false;
input.nextcharClass(c,true);
continue;
}
retract();
break;
}
state=start_state;
String s=input.copyReplaceUnicodeEscapes(needs_escape);
if (maybe_reserved) {
Integer i=reservedWord.get(s);
if (i != null) return makeToken((int)i);
}
return makeToken(IDENTIFIER_TOKEN,s);
}
case zero_state:
switch (nextchar()) {
case 'x':
case 'X':
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
state=hexinteger_state;
break;
default :
state=start_state;
error(kError_Lexical_General);
}
continue;
case '.':
state=decimal_state;
continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimalinteger_state;
continue;
case 'E':
case 'e':
state=exponentstart_state;
continue;
case 'd':
case 'm':
case 'i':
case 'u':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case hexinteger_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
state=hexinteger_state;
continue;
case 'u':
case 'i':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case dot_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimal_state;
continue;
case '.':
state=start_state;
if (nextchar() == '.') return makeToken(TRIPLEDOT_TOKEN);
retract();
return makeToken(DOUBLEDOT_TOKEN);
case '<':
state=start_state;
return makeToken(DOTLESSTHAN_TOKEN);
default :
retract();
state=start_state;
return makeToken(DOT_TOKEN);
}
case decimalinteger_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimalinteger_state;
continue;
case '.':
state=decimal_state;
continue;
case 'd':
case 'm':
case 'u':
case 'i':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
case 'E':
case 'e':
state=exponentstart_state;
continue;
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case decimal_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimal_state;
continue;
case 'd':
case 'm':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
case 'E':
case 'e':
state=exponentstart_state;
continue;
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case exponentstart_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case '+':
case '-':
state=exponent_state;
continue;
default :
error(kError_Lexical_General);
state=start_state;
continue;
}
case exponent_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=exponent_state;
continue;
case 'd':
case 'm':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case slash_state:
{
c=nextchar();
switch (c) {
case '/':
state=start_state;
line_comment: while ((c=nextchar()) != 0) {
if (c == '\r' || c == '\n') {
isFirstTokenOnLine=true;
if (save_comments == false) {
break line_comment;
}
retract();
return makeToken(SLASHSLASHCOMMENT_TOKEN,input.copyReplaceUnicodeEscapes());
}
}
continue;
case '*':
if (save_comments == false) {
block_comment: while ((c=nextchar()) != 0) {
if (c == '\r' || c == '\n') isFirstTokenOnLine=true;
if (c == '*') {
c=nextchar();
if (c == '/') {
break block_comment;
}
retract();
}
}
state=start_state;
}
 else {
if (blockcommentbuf == null) blockcommentbuf=new StringBuilder();
blockcommentbuf.append("String_Node_Str");
state=blockcommentstart_state;
}
continue;
case '>':
if (inXML > 0) {
state=start_state;
return makeToken(XMLTAGENDEND_TOKEN);
}
default :
int lb=currentToken.id;
if (lb == IDENTIFIER_TOKEN || lb == NUMBERLITERAL_TOKEN || lb == RIGHTPAREN_TOKEN || lb == RIGHTBRACE_TOKEN || lb == RIGHTBRACKET_TOKEN) {
state=start_state;
if (c == '=') return makeToken(DIVASSIGN_TOKEN);
retract();
return makeToken(DIV_TOKEN);
}
state=slashregexp_state;
retract();
continue;
}
}
case slashregexp_state:
switch (nextchar()) {
case '\\':
nextchar();
continue;
case '/':
regexp_flags=0;
state=regexp_state;
continue;
case 0:
case '\n':
case '\r':
error(kError_Lexical_General);
state=start_state;
continue;
default :
state=slashregexp_state;
continue;
}
case regexp_state:
c=nextchar();
switch (c) {
case 'g':
if ((regexp_flags & 0x01) == 0) {
regexp_flags|=0x01;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 'i':
if ((regexp_flags & 0x02) == 0) {
regexp_flags|=0x02;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 'm':
if ((regexp_flags & 0x04) == 0) {
regexp_flags|=0x04;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 's':
if ((regexp_flags & 0x08) == 0) {
regexp_flags|=0x08;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 'x':
if ((regexp_flags & 0x10) == 0) {
regexp_flags|=0x10;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
default :
if (Character.isJavaIdentifierPart(c)) {
error(kError_Lexical_General);
state=start_state;
continue;
}
retract();
state=start_state;
return makeToken(REGEXPLITERAL_TOKEN,input.copyReplaceUnicodeEscapes());
}
case xmlcommentorcdatastart_state:
switch (nextchar()) {
case '[':
if (nextchar() == 'C' && nextchar() == 'D' && nextchar() == 'A' && nextchar() == 'T' && nextchar() == 'A' && nextchar() == '[') {
state=xmlcdata_state;
continue;
}
break;
case '-':
if (nextchar() == '-') {
state=xmlcomment_state;
continue;
}
}
error(kError_Lexical_General);
state=start_state;
continue;
case xmlcdata_state:
switch (nextchar()) {
case ']':
if (nextchar() == ']' && nextchar() == '>') {
state=start_state;
return makeToken(XMLMARKUP_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos()));
}
continue;
case 0:
error(kError_Lexical_General);
state=start_state;
}
continue;
case xmlcomment_state:
while ((c=nextchar()) != '-' && c != 0) ;
if (c == '-' && nextchar() != '-') {
continue;
}
if (nextchar() == '>') {
state=start_state;
return makeToken(XMLMARKUP_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos()));
}
error(kError_Lexical_General);
state=start_state;
continue;
case xmlpi_state:
while ((c=nextchar()) != '?' && c != 0) ;
if (c == '?' && nextchar() == '>') {
state=start_state;
return makeToken(XMLMARKUP_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos()));
}
if (c == 0) {
error(kError_Lexical_General);
state=start_state;
}
continue;
case xmltext_state:
{
switch (nextchar()) {
case '<':
case '{':
{
retract();
String xmltext=input.substringReplaceUnicodeEscapes(startofxml,pos());
if (xmltext != null) {
state=start_state;
return makeToken(XMLTEXT_TOKEN,xmltext);
}
 else {
switch (nextchar()) {
case '<':
switch (nextchar()) {
case '/':
state=start_state;
return makeToken(XMLTAGSTARTEND_TOKEN);
case '!':
state=xmlcommentorcdatastart_state;
continue;
case '?':
state=xmlpi_state;
continue;
default :
retract();
state=start_state;
return makeToken(LESSTHAN_TOKEN);
}
case '{':
state=start_state;
return makeToken(LEFTBRACE_TOKEN);
}
}
}
case 0:
state=start_state;
return makeToken(EOS_TOKEN);
}
continue;
}
case xmlliteral_state:
switch (nextchar()) {
case '{':
return makeToken(XMLPART_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos() - 1));
case '<':
if (nextchar() == '/') {
--level;
nextchar();
mark();
retract();
state=endxmlname_state;
}
 else {
++level;
state=xmlliteral_state;
}
continue;
case '/':
if (nextchar() == '>') {
--level;
if (level == 0) {
state=start_state;
return makeToken(XMLLITERAL_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos() + 1));
}
}
continue;
case 0:
retract();
error(kError_Lexical_NoMatchingTag);
state=start_state;
continue;
default :
continue;
}
case endxmlname_state:
c=nextchar();
if (Character.isJavaIdentifierPart(c) || c == ':') {
continue;
}
switch (c) {
case '{':
{
String xmltext=input.substringReplaceUnicodeEscapes(startofxml,pos() - 1);
return makeToken(XMLPART_TOKEN,xmltext);
}
case '>':
retract();
nextchar();
if (level == 0) {
String xmltext=input.substringReplaceUnicodeEscapes(startofxml,pos() + 1);
state=start_state;
return makeToken(XMLLITERAL_TOKEN,xmltext);
}
state=xmlliteral_state;
continue;
default :
state=xmlliteral_state;
continue;
}
case blockcommentstart_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '*':
if (nextchar() == '/') {
state=start_state;
return makeToken(BLOCKCOMMENT_TOKEN,new String());
}
retract();
state=doccomment_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
case '\n':
case '\r':
isFirstTokenOnLine=true;
default :
state=blockcomment_state;
continue;
}
}
case doccomment_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '*':
state=doccommentstar_state;
continue;
case '@':
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
if (doctagname.length() > 0) {
doctextbuf.append("String_Node_Str").append(doctagname).append("String_Node_Str");
}
doctagname="String_Node_Str";
state=doccommenttag_state;
continue;
case '\r':
case '\n':
isFirstTokenOnLine=true;
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
doctextbuf.append('\n');
state=doccomment_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
doctextbuf.append((char)(c));
state=doccomment_state;
continue;
}
}
case doccommentstar_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '/':
{
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
if (doctagname.length() > 0) {
doctextbuf.append("String_Node_Str").append(doctagname).append("String_Node_Str");
}
String doctext=doctextbuf.toString();
state=start_state;
return makeToken(DOCCOMMENT_TOKEN,doctext);
}
case '*':
state=doccommentstar_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
state=doccomment_state;
continue;
}
}
case doccommenttag_state:
{
c=nextchar();
switch (c) {
case '*':
state=doccommentstar_state;
continue;
case ' ':
case '\t':
case '\r':
case '\n':
{
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
while ((c=nextchar()) == ' ' || c == '\t') ;
retract();
if (doctagname.length() > 0) {
doctextbuf.append("String_Node_Str").append(doctagname).append("String_Node_Str");
}
state=doccomment_state;
continue;
}
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
doctagname+=(char)(c);
continue;
}
}
case doccommentvalue_state:
switch (nextchar()) {
case '*':
state=doccommentstar_state;
continue;
case '@':
state=doccommenttag_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
state=doccomment_state;
continue;
}
case blockcomment_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '*':
state=blockcommentstar_state;
continue;
case '\r':
case '\n':
isFirstTokenOnLine=true;
state=blockcomment_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
state=blockcomment_state;
continue;
}
}
case blockcommentstar_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '/':
{
state=start_state;
String blocktext=blockcommentbuf.toString();
return makeToken(BLOCKCOMMENT_TOKEN,blocktext);
}
case '*':
state=blockcommentstar_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
state=blockcomment_state;
continue;
}
}
case error_state:
error(kError_Lexical_General);
skiperror();
state=start_state;
continue;
default :
error("String_Node_Str");
state=start_state;
return makeToken(EOS_TOKEN);
}
}
}

Fixed version (ground truth):
public int nexttoken(boolean resetState){
  String doctagname="String_Node_Str";
  StringBuilder doctextbuf=null;
  int startofxml=pos();
  StringBuilder blockcommentbuf=null;
  char regexp_flags=0;
  boolean maybe_reserved=false;
  char c=0;
  if (resetState) {
    isFirstTokenOnLine=false;
  }
  while (true) {
    if (debug) {
      System.out.println("String_Node_Str" + state + "String_Node_Str"+ pos());
    }
switch (state) {
case start_state:
{
        c=nextchar();
        mark();
switch (c) {
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'g':
case 'h':
case 'i':
case 'j':
case 'k':
case 'l':
case 'm':
case 'n':
case 'o':
case 'p':
case 'q':
case 'r':
case 's':
case 't':
case 'u':
case 'v':
case 'w':
case 'x':
case 'y':
case 'z':
          maybe_reserved=true;
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
case 'G':
case 'H':
case 'I':
case 'J':
case 'K':
case 'L':
case 'M':
case 'N':
case 'O':
case 'P':
case 'Q':
case 'R':
case 'S':
case 'T':
case 'U':
case 'V':
case 'W':
case 'X':
case 'Y':
case 'Z':
case '_':
case '$':
        state=A_state;
      continue;
case 0xffef:
    if (nextchar() == 0xffffffbb && nextchar() == 0xffffffbf) {
      state=start_state;
    }
 else {
      state=error_state;
    }
  continue;
case '@':
return makeToken(ATSIGN_TOKEN);
case '\'':
case '\"':
{
char startquote=(char)c;
boolean needs_escape=false;
while ((c=nextchar()) != startquote) {
  if (c == '\\') {
    needs_escape=true;
    c=nextchar();
    if (c == '\r')     c=nextchar();
    if (c == '\n')     continue;
  }
 else   if (c == '\r' || c == '\n') {
    if (startquote == '\'')     error(kError_Lexical_LineTerminatorInSingleQuotedStringLiteral);
 else     error(kError_Lexical_LineTerminatorInDoubleQuotedStringLiteral);
    break;
  }
 else   if (c == 0) {
    error(kError_Lexical_EndOfStreamInStringLiteral);
    return makeToken(EOS_TOKEN);
  }
}
return makeToken(STRINGLITERAL_TOKEN,input.copyReplaceStringEscapes(needs_escape));
}
case '-':
switch (nextchar()) {
case '-':
return makeToken(MINUSMINUS_TOKEN);
case '=':
return makeToken(MINUSASSIGN_TOKEN);
default :
retract();
return makeToken(MINUS_TOKEN);
}
case '!':
if (nextchar() == '=') {
if (nextchar() == '=') return makeToken(STRICTNOTEQUALS_TOKEN);
retract();
return makeToken(NOTEQUALS_TOKEN);
}
retract();
return makeToken(NOT_TOKEN);
case '%':
switch (nextchar()) {
case '=':
return makeToken(MODULUSASSIGN_TOKEN);
default :
retract();
return makeToken(MODULUS_TOKEN);
}
case '&':
c=nextchar();
if (c == '=') return makeToken(BITWISEANDASSIGN_TOKEN);
if (c == '&') {
if (nextchar() == '=') return makeToken(LOGICALANDASSIGN_TOKEN);
retract();
return makeToken(LOGICALAND_TOKEN);
}
retract();
return makeToken(BITWISEAND_TOKEN);
case '#':
if (HAS_HASHPRAGMAS) {
return makeToken(USE_TOKEN);
}
state=error_state;
continue;
case '(':
return makeToken(LEFTPAREN_TOKEN);
case ')':
return makeToken(RIGHTPAREN_TOKEN);
case '*':
if (nextchar() == '=') return makeToken(MULTASSIGN_TOKEN);
retract();
return makeToken(MULT_TOKEN);
case ',':
return makeToken(COMMA_TOKEN);
case '.':
state=dot_state;
continue;
case '/':
state=slash_state;
continue;
case ':':
if (nextchar() == ':') {
return makeToken(DOUBLECOLON_TOKEN);
}
retract();
return makeToken(COLON_TOKEN);
case ';':
return makeToken(SEMICOLON_TOKEN);
case '?':
return makeToken(QUESTIONMARK_TOKEN);
case '[':
return makeToken(LEFTBRACKET_TOKEN);
case ']':
return makeToken(RIGHTBRACKET_TOKEN);
case '^':
if (nextchar() == '=') return makeToken(BITWISEXORASSIGN_TOKEN);
retract();
return makeToken(BITWISEXOR_TOKEN);
case '{':
return makeToken(LEFTBRACE_TOKEN);
case '|':
c=nextchar();
if (c == '=') return makeToken(BITWISEORASSIGN_TOKEN);
if (c == '|') {
if (nextchar() == '=') return makeToken(LOGICALORASSIGN_TOKEN);
retract();
return makeToken(LOGICALOR_TOKEN);
}
retract();
return makeToken(BITWISEOR_TOKEN);
case '}':
return makeToken(RIGHTBRACE_TOKEN);
case '~':
return makeToken(BITWISENOT_TOKEN);
case '+':
c=nextchar();
if (c == '+') return makeToken(PLUSPLUS_TOKEN);
if (c == '=') return makeToken(PLUSASSIGN_TOKEN);
retract();
return makeToken(PLUS_TOKEN);
case '<':
switch (nextchar()) {
case '<':
if (nextchar() == '=') return makeToken(LEFTSHIFTASSIGN_TOKEN);
retract();
return makeToken(LEFTSHIFT_TOKEN);
case '=':
return makeToken(LESSTHANOREQUALS_TOKEN);
case '/':
return makeToken(XMLTAGSTARTEND_TOKEN);
case '!':
state=xmlcommentorcdatastart_state;
continue;
case '?':
state=xmlpi_state;
continue;
}
retract();
return makeToken(LESSTHAN_TOKEN);
case '=':
if (nextchar() == '=') {
if (nextchar() == '=') return makeToken(STRICTEQUALS_TOKEN);
retract();
return makeToken(EQUALS_TOKEN);
}
retract();
return makeToken(ASSIGN_TOKEN);
case '>':
state=start_state;
switch (nextchar()) {
case '>':
switch (nextchar()) {
case '>':
if (nextchar() == '=') return makeToken(UNSIGNEDRIGHTSHIFTASSIGN_TOKEN);
retract();
return makeToken(UNSIGNEDRIGHTSHIFT_TOKEN);
case '=':
return makeToken(RIGHTSHIFTASSIGN_TOKEN);
default :
retract();
return makeToken(RIGHTSHIFT_TOKEN);
}
case '=':
return makeToken(GREATERTHANOREQUALS_TOKEN);
}
retract();
return makeToken(GREATERTHAN_TOKEN);
case '0':
state=zero_state;
continue;
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimalinteger_state;
continue;
case ' ':
case '\t':
case 0x000b:
case 0x000c:
case 0x0085:
case 0x00a0:
continue;
case '\n':
case '\r':
isFirstTokenOnLine=true;
continue;
case 0:
return makeToken(EOS_TOKEN);
default :
switch (input.nextcharClass((char)c,true)) {
case Lu:
case Ll:
case Lt:
case Lm:
case Lo:
case Nl:
maybe_reserved=false;
state=A_state;
continue;
case Zs:
case Cc:
case Cf:
continue;
case Zl:
case Zp:
isFirstTokenOnLine=true;
continue;
default :
state=error_state;
continue;
}
}
}
case A_state:
{
boolean needs_escape=c == '\\';
while (true) {
c=nextchar();
if (c >= 'a' && c <= 'z') {
continue;
}
if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '$' || c == '_') {
maybe_reserved=false;
continue;
}
if (c <= 0x7f) {
if (c == '\\') {
needs_escape=true;
}
 else {
retract();
break;
}
}
switch (input.nextcharClass(c,false)) {
case Lu:
case Ll:
case Lt:
case Lm:
case Lo:
case Nl:
case Mn:
case Mc:
case Nd:
case Pc:
maybe_reserved=false;
input.nextcharClass(c,true);
continue;
}
retract();
break;
}
state=start_state;
String s=input.copyReplaceUnicodeEscapes(needs_escape);
if (maybe_reserved) {
Integer i=reservedWord.get(s);
if (i != null) return makeToken((int)i);
}
return makeToken(IDENTIFIER_TOKEN,s);
}
case zero_state:
switch (nextchar()) {
case 'x':
case 'X':
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
state=hexinteger_state;
break;
default :
state=start_state;
error(kError_Lexical_General);
}
continue;
case '.':
state=decimal_state;
continue;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimalinteger_state;
continue;
case 'E':
case 'e':
state=exponentstart_state;
continue;
case 'd':
case 'm':
case 'i':
case 'u':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case hexinteger_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case 'a':
case 'b':
case 'c':
case 'd':
case 'e':
case 'f':
case 'A':
case 'B':
case 'C':
case 'D':
case 'E':
case 'F':
state=hexinteger_state;
continue;
case 'u':
case 'i':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case dot_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimal_state;
continue;
case '.':
state=start_state;
if (nextchar() == '.') return makeToken(TRIPLEDOT_TOKEN);
retract();
return makeToken(DOUBLEDOT_TOKEN);
case '<':
state=start_state;
return makeToken(DOTLESSTHAN_TOKEN);
default :
retract();
state=start_state;
return makeToken(DOT_TOKEN);
}
case decimalinteger_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimalinteger_state;
continue;
case '.':
state=decimal_state;
continue;
case 'd':
case 'm':
case 'u':
case 'i':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
case 'E':
case 'e':
state=exponentstart_state;
continue;
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case decimal_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=decimal_state;
continue;
case 'd':
case 'm':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
case 'E':
case 'e':
state=exponentstart_state;
continue;
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case exponentstart_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case '+':
case '-':
state=exponent_state;
continue;
default :
error(kError_Lexical_General);
state=start_state;
continue;
}
case exponent_state:
switch (nextchar()) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
state=exponent_state;
continue;
case 'd':
case 'm':
if (!ctx.statics.es4_numerics) retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
default :
retract();
state=start_state;
return makeToken(NUMBERLITERAL_TOKEN,input.copy());
}
case slash_state:
{
c=nextchar();
switch (c) {
case '/':
state=start_state;
line_comment: while ((c=nextchar()) != 0) {
if (c == '\r' || c == '\n') {
isFirstTokenOnLine=true;
if (save_comments == false) {
break line_comment;
}
retract();
return makeCommentToken(SLASHSLASHCOMMENT_TOKEN,input.copyReplaceUnicodeEscapes());
}
}
continue;
case '*':
if (save_comments == false) {
block_comment: while ((c=nextchar()) != 0) {
if (c == '\r' || c == '\n') isFirstTokenOnLine=true;
if (c == '*') {
c=nextchar();
if (c == '/') {
break block_comment;
}
retract();
}
}
state=start_state;
}
 else {
if (blockcommentbuf == null) blockcommentbuf=new StringBuilder();
blockcommentbuf.append("String_Node_Str");
state=blockcommentstart_state;
}
continue;
case '>':
if (inXML > 0) {
state=start_state;
return makeToken(XMLTAGENDEND_TOKEN);
}
default :
int lb=currentToken.lookback;
if (lb == IDENTIFIER_TOKEN || lb == NUMBERLITERAL_TOKEN || lb == RIGHTPAREN_TOKEN || lb == RIGHTBRACE_TOKEN || lb == RIGHTBRACKET_TOKEN) {
state=start_state;
if (c == '=') return makeToken(DIVASSIGN_TOKEN);
retract();
return makeToken(DIV_TOKEN);
}
state=slashregexp_state;
retract();
continue;
}
}
case slashregexp_state:
switch (nextchar()) {
case '\\':
nextchar();
continue;
case '/':
regexp_flags=0;
state=regexp_state;
continue;
case 0:
case '\n':
case '\r':
error(kError_Lexical_General);
state=start_state;
continue;
default :
state=slashregexp_state;
continue;
}
case regexp_state:
c=nextchar();
switch (c) {
case 'g':
if ((regexp_flags & 0x01) == 0) {
regexp_flags|=0x01;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 'i':
if ((regexp_flags & 0x02) == 0) {
regexp_flags|=0x02;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 'm':
if ((regexp_flags & 0x04) == 0) {
regexp_flags|=0x04;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 's':
if ((regexp_flags & 0x08) == 0) {
regexp_flags|=0x08;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
case 'x':
if ((regexp_flags & 0x10) == 0) {
regexp_flags|=0x10;
continue;
}
error(kError_Lexical_General);
state=start_state;
continue;
default :
if (Character.isJavaIdentifierPart(c)) {
error(kError_Lexical_General);
state=start_state;
continue;
}
retract();
state=start_state;
return makeToken(REGEXPLITERAL_TOKEN,input.copyReplaceUnicodeEscapes());
}
case xmlcommentorcdatastart_state:
switch (nextchar()) {
case '[':
if (nextchar() == 'C' && nextchar() == 'D' && nextchar() == 'A' && nextchar() == 'T' && nextchar() == 'A' && nextchar() == '[') {
state=xmlcdata_state;
continue;
}
break;
case '-':
if (nextchar() == '-') {
state=xmlcomment_state;
continue;
}
}
error(kError_Lexical_General);
state=start_state;
continue;
case xmlcdata_state:
switch (nextchar()) {
case ']':
if (nextchar() == ']' && nextchar() == '>') {
state=start_state;
return makeToken(XMLMARKUP_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos()));
}
continue;
case 0:
error(kError_Lexical_General);
state=start_state;
}
continue;
case xmlcomment_state:
while ((c=nextchar()) != '-' && c != 0) ;
if (c == '-' && nextchar() != '-') {
continue;
}
if (nextchar() == '>') {
state=start_state;
return makeToken(XMLMARKUP_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos()));
}
error(kError_Lexical_General);
state=start_state;
continue;
case xmlpi_state:
while ((c=nextchar()) != '?' && c != 0) ;
if (c == '?' && nextchar() == '>') {
state=start_state;
return makeToken(XMLMARKUP_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos()));
}
if (c == 0) {
error(kError_Lexical_General);
state=start_state;
}
continue;
case xmltext_state:
{
switch (nextchar()) {
case '<':
case '{':
{
retract();
String xmltext=input.substringReplaceUnicodeEscapes(startofxml,pos());
if (xmltext != null) {
state=start_state;
return makeToken(XMLTEXT_TOKEN,xmltext);
}
 else {
switch (nextchar()) {
case '<':
switch (nextchar()) {
case '/':
state=start_state;
return makeToken(XMLTAGSTARTEND_TOKEN);
case '!':
state=xmlcommentorcdatastart_state;
continue;
case '?':
state=xmlpi_state;
continue;
default :
retract();
state=start_state;
return makeToken(LESSTHAN_TOKEN);
}
case '{':
state=start_state;
return makeToken(LEFTBRACE_TOKEN);
}
}
}
case 0:
state=start_state;
return makeToken(EOS_TOKEN);
}
continue;
}
case xmlliteral_state:
switch (nextchar()) {
case '{':
return makeToken(XMLPART_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos() - 1));
case '<':
if (nextchar() == '/') {
--level;
nextchar();
mark();
retract();
state=endxmlname_state;
}
 else {
++level;
state=xmlliteral_state;
}
continue;
case '/':
if (nextchar() == '>') {
--level;
if (level == 0) {
state=start_state;
return makeToken(XMLLITERAL_TOKEN,input.substringReplaceUnicodeEscapes(startofxml,pos() + 1));
}
}
continue;
case 0:
retract();
error(kError_Lexical_NoMatchingTag);
state=start_state;
continue;
default :
continue;
}
case endxmlname_state:
c=nextchar();
if (Character.isJavaIdentifierPart(c) || c == ':') {
continue;
}
switch (c) {
case '{':
{
String xmltext=input.substringReplaceUnicodeEscapes(startofxml,pos() - 1);
return makeToken(XMLPART_TOKEN,xmltext);
}
case '>':
retract();
nextchar();
if (level == 0) {
String xmltext=input.substringReplaceUnicodeEscapes(startofxml,pos() + 1);
state=start_state;
return makeToken(XMLLITERAL_TOKEN,xmltext);
}
state=xmlliteral_state;
continue;
default :
state=xmlliteral_state;
continue;
}
case blockcommentstart_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '*':
if (nextchar() == '/') {
state=start_state;
return makeCommentToken(BLOCKCOMMENT_TOKEN,new String());
}
retract();
state=doccomment_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
case '\n':
case '\r':
isFirstTokenOnLine=true;
default :
state=blockcomment_state;
continue;
}
}
case doccomment_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '*':
state=doccommentstar_state;
continue;
case '@':
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
if (doctagname.length() > 0) {
doctextbuf.append("String_Node_Str").append(doctagname).append("String_Node_Str");
}
doctagname="String_Node_Str";
state=doccommenttag_state;
continue;
case '\r':
case '\n':
isFirstTokenOnLine=true;
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
doctextbuf.append('\n');
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
doctextbuf.append((char)(c));
continue;
}
}
case doccommentstar_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '/':
{
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
if (doctagname.length() > 0) {
doctextbuf.append("String_Node_Str").append(doctagname).append("String_Node_Str");
}
String doctext=doctextbuf.toString();
state=start_state;
return makeCommentToken(DOCCOMMENT_TOKEN,doctext);
}
case '*':
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
state=doccomment_state;
continue;
}
}
case doccommenttag_state:
{
c=nextchar();
switch (c) {
case '*':
state=doccommentstar_state;
continue;
case ' ':
case '\t':
case '\r':
case '\n':
{
if (doctextbuf == null) doctextbuf=getDocTextBuffer(doctagname);
while ((c=nextchar()) == ' ' || c == '\t') ;
retract();
if (doctagname.length() > 0) {
doctextbuf.append("String_Node_Str").append(doctagname).append("String_Node_Str");
}
state=doccomment_state;
continue;
}
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
doctagname+=(char)(c);
continue;
}
}
case doccommentvalue_state:
switch (nextchar()) {
case '*':
state=doccommentstar_state;
continue;
case '@':
state=doccommenttag_state;
continue;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
continue;
default :
state=doccomment_state;
continue;
}
case blockcomment_state:
{
c=nextchar();
blockcommentbuf.append(c);
switch (c) {
case '*':
c=nextchar();
if (c == '/') {
state=start_state;
blockcommentbuf.append(c);
String blocktext=blockcommentbuf.toString();
return makeCommentToken(BLOCKCOMMENT_TOKEN,blocktext);
}
retract();
break;
case '\r':
case '\n':
isFirstTokenOnLine=true;
break;
case 0:
error(kError_BlockCommentNotTerminated);
state=start_state;
break;
}
continue;
}
case error_state:
error(kError_Lexical_General);
skiperror();
state=start_state;
continue;
default :
error("String_Node_Str");
state=start_state;
return makeToken(EOS_TOKEN);
}
}
}