Program: 83717
Project: 83717
Buggy source:
public static IToken tokenize(String code){
  int len=code.length();
  StringBuilder buf=new StringBuilder(20);
  IToken first=new InferredSemicolon(0,0);
  IToken prev=first;
  int start=0;
  int lineNumber=1;
  char l=0;
  char c=0;
  int type=0;
  int subtype=0;
  boolean addToken=false;
  boolean reparse=true;
  boolean string=false;
  for (int i=0; i < len; ++i, l=c) {
    c=code.charAt(i);
    if (type == 0) {
      start=i;
      if (c == '\n') {
        lineNumber++;
        continue;
      }
      if (c <= ' ') {
        continue;
      }
      if (string && c == '}') {
        type=STRING_2;
        subtype=STRING_PART;
        continue;
      }
      int m=getMode(c,code,i);
      type=m & 0xFFFF;
      subtype=m & 0xFFFF0000;
    }
switch (type) {
case IDENTIFIER:
      if (subtype == MOD_DOTS) {
        if (c == '.') {
          buf.append(c);
        }
 else {
          addToken=true;
          reparse=true;
        }
      }
 else       if (c == '_' || c == '$') {
        subtype=MOD_SYMBOL | MOD_LETTER;
        buf.append(c);
      }
 else {
        boolean letter=(subtype & MOD_LETTER) != 0;
        boolean symbol=(subtype & MOD_SYMBOL) != 0;
        if (letter) {
          if (isIdentifierPart(c)) {
            subtype=MOD_LETTER;
            buf.append(c);
            continue;
          }
        }
        if (symbol) {
          if (isIdentifierSymbol(c)) {
            subtype=MOD_SYMBOL;
            buf.append(c);
            continue;
          }
        }
        addToken=true;
      }
    break;
case SYMBOL:
  buf.append(c);
addToken=true;
reparse=false;
break;
case BRACKET:
buf.append(c);
addToken=true;
reparse=false;
break;
case COMMENT:
if (subtype == MOD_LINE) {
if (c == '\n') {
type=0;
lineNumber++;
continue;
}
}
 else if (subtype == MOD_BLOCK) {
if (c == '\n') {
lineNumber++;
}
 else if (l == '*' && c == '/') {
type=0;
continue;
}
}
break;
case INT:
case LONG:
if (c == '.') {
type=FLOAT;
buf.append('.');
}
 else if (c == 'l' || c == 'L') {
type=LONG;
addToken=true;
reparse=false;
}
 else if (subtype == MOD_DEC) {
if (isDigit(c)) {
buf.append(c);
}
 else if (c == 'f' || c == 'F') {
type=FLOAT;
addToken=true;
reparse=false;
}
 else if (c == 'd' || c == 'D') {
type=DOUBLE;
addToken=true;
reparse=false;
}
 else {
addToken=true;
}
}
 else if (subtype == MOD_BIN) {
if (c == 'b' || isBinDigit(c)) {
buf.append(c);
}
 else {
addToken=true;
}
}
 else if (subtype == MOD_OCT) {
if (isOctDigit(c)) {
buf.append(c);
}
 else {
addToken=true;
}
}
 else if (subtype == MOD_HEX) {
if (c == 'x' || isHexDigit(c)) {
buf.append(c);
}
 else {
addToken=true;
}
}
break;
case FLOAT:
case DOUBLE:
if (c == 'x') {
subtype=MOD_HEX;
buf.append(c);
}
 else if (c == 'f' || c == 'F') {
addToken=true;
reparse=false;
}
 else if (c == 'd' || c == 'D') {
type=DOUBLE;
addToken=true;
reparse=false;
}
 else if (isDigit(c) || c == '.' || c == 'e') {
buf.append(c);
}
 else {
addToken=true;
}
break;
case STRING:
if (c == '"' && buf.length() > 0) {
buf.append('"');
addToken=true;
reparse=false;
}
 else if (c == '\\' && appendEscape(buf,code.charAt(i + 1))) {
i++;
continue;
}
 else if (c != '\t') {
buf.append(c);
}
break;
case STRING_2:
if (c == '"' && (buf.length() > 1 || string)) {
if (!string && buf.charAt(0) == '@') {
subtype=STRING_2;
}
 else {
subtype=STRING_END;
}
string=false;
addToken=true;
reparse=false;
break;
}
 else if (c == '\\' && appendEscape(buf,code.charAt(i + 1))) {
i++;
continue;
}
 else if (c == '$' && code.charAt(i + 1) == '{') {
i+=2;
if (buf.charAt(0) == '@') {
subtype=STRING_START;
}
 else {
subtype=STRING_PART;
}
addToken=true;
string=true;
}
 else if (c != '\t') {
buf.append(c);
}
break;
case CHAR:
if (c == '\'' && buf.length() > 0) {
buf.append('\'');
addToken=true;
reparse=false;
}
 else if (c == '\\' && appendEscape(buf,code.charAt(i + 1))) {
i++;
continue;
}
 else if (c != '\t') {
buf.append(c);
}
break;
}
if (addToken) {
prev=addToken(prev,buf,type | subtype,lineNumber,start);
addToken=false;
type=0;
if (reparse) {
i--;
}
 else {
reparse=true;
}
}
}
if (buf.length() > 0) {
addToken(prev,buf,type | subtype,lineNumber,start);
}
return first.getNext();
}

Fixed version (ground truth):
public static IToken tokenize(String code){
  int len=code.length();
  StringBuilder buf=new StringBuilder(20);
  IToken first=new InferredSemicolon(0,0);
  IToken prev=first;
  int start=0;
  int lineNumber=1;
  char l=0;
  char c=0;
  int type=0;
  int subtype=0;
  boolean addToken=false;
  boolean reparse=true;
  boolean string=false;
  for (int i=0; i < len; ++i, l=c) {
    c=code.charAt(i);
    if (type == 0) {
      start=i;
      if (c == '\n') {
        lineNumber++;
        continue;
      }
      if (c <= ' ') {
        continue;
      }
      if (string && c == '}') {
        type=STRING_2;
        subtype=STRING_PART;
        continue;
      }
      int m=getMode(c,code,i);
      type=m & 0xFFFF;
      subtype=m & 0xFFFF0000;
    }
switch (type) {
case IDENTIFIER:
      if (subtype == MOD_DOTS) {
        if (c == '.') {
          buf.append(c);
        }
 else {
          addToken=true;
          reparse=true;
        }
      }
 else       if (c == '_' || c == '$') {
        subtype=MOD_SYMBOL | MOD_LETTER;
        buf.append(c);
      }
 else {
        boolean letter=(subtype & MOD_LETTER) != 0;
        boolean symbol=(subtype & MOD_SYMBOL) != 0;
        if (letter) {
          if (isIdentifierPart(c)) {
            subtype=MOD_LETTER;
            buf.append(c);
            continue;
          }
        }
        if (symbol) {
          if (isIdentifierSymbol(c)) {
            subtype=MOD_SYMBOL;
            buf.append(c);
            continue;
          }
        }
        addToken=true;
      }
    break;
case SYMBOL:
  buf.append(c);
addToken=true;
reparse=false;
break;
case BRACKET:
buf.append(c);
addToken=true;
reparse=false;
break;
case COMMENT:
if (subtype == MOD_LINE) {
if (c == '\n') {
type=0;
lineNumber++;
continue;
}
}
 else if (subtype == MOD_BLOCK) {
if (c == '\n') {
lineNumber++;
}
 else if (l == '*' && c == '/') {
type=0;
continue;
}
}
break;
case INT:
case LONG:
if (c == '.') {
type=FLOAT;
buf.append('.');
}
 else if (c == 'l' || c == 'L') {
type=LONG;
addToken=true;
reparse=false;
}
 else if (subtype == MOD_DEC) {
if (isDigit(c)) {
buf.append(c);
}
 else if (c == 'f' || c == 'F') {
type=FLOAT;
addToken=true;
reparse=false;
}
 else if (c == 'd' || c == 'D') {
type=DOUBLE;
addToken=true;
reparse=false;
}
 else {
addToken=true;
}
}
 else if (subtype == MOD_BIN) {
if (c == 'b' || isBinDigit(c)) {
buf.append(c);
}
 else {
addToken=true;
}
}
 else if (subtype == MOD_OCT) {
if (isOctDigit(c)) {
buf.append(c);
}
 else {
addToken=true;
}
}
 else if (subtype == MOD_HEX) {
if (c == 'x' || isHexDigit(c)) {
buf.append(c);
}
 else {
addToken=true;
}
}
break;
case FLOAT:
case DOUBLE:
if (c == 'x') {
subtype=MOD_HEX;
buf.append(c);
}
 else if (c == 'f' || c == 'F') {
addToken=true;
reparse=false;
}
 else if (c == 'd' || c == 'D') {
type=DOUBLE;
addToken=true;
reparse=false;
}
 else if (isDigit(c) || c == '.' || c == 'e') {
buf.append(c);
}
 else {
addToken=true;
}
break;
case STRING:
if (c == '"' && buf.length() > 0) {
buf.append('"');
addToken=true;
reparse=false;
}
 else if (c == '\\' && appendEscape(buf,code.charAt(i + 1))) {
i++;
continue;
}
 else if (c != '\t') {
buf.append(c);
}
break;
case STRING_2:
if (c == '"' && (buf.length() > 1 || string)) {
if (!string && buf.charAt(0) == '@') {
subtype=STRING_2;
}
 else {
subtype=STRING_END;
}
string=false;
addToken=true;
reparse=false;
break;
}
 else if (c == '\\' && appendEscape(buf,code.charAt(i + 1))) {
i++;
continue;
}
 else if (c == '$' && code.charAt(i + 1) == '{') {
i+=2;
if (buf.length() == 0 || buf.charAt(0) == '}') {
subtype=STRING_PART;
}
 else {
subtype=STRING_START;
}
addToken=true;
string=true;
}
 else if (c != '\t') {
buf.append(c);
}
break;
case CHAR:
if (c == '\'' && buf.length() > 0) {
buf.append('\'');
addToken=true;
reparse=false;
}
 else if (c == '\\' && appendEscape(buf,code.charAt(i + 1))) {
i++;
continue;
}
 else if (c != '\t') {
buf.append(c);
}
break;
}
if (addToken) {
prev=addToken(prev,buf,type | subtype,lineNumber,start);
addToken=false;
type=0;
if (reparse) {
i--;
}
 else {
reparse=true;
}
}
}
if (buf.length() > 0) {
addToken(prev,buf,type | subtype,lineNumber,start);
}
return first.getNext();
}