Buggy code:
public class MockTokenizer extends Tokenizer { public static final CharacterRunAutomaton WHITESPACE = new CharacterRunAutomaton ( new RegExp ( "[^<seq2seq4repair_space>\t\r\n]+" ) . toAutomaton ( ) ) ; public static final CharacterRunAutomaton KEYWORD = new CharacterRunAutomaton ( new RegExp ( ".*" ) . toAutomaton ( ) ) ; public static final CharacterRunAutomaton SIMPLE = new CharacterRunAutomaton ( new RegExp ( "[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԥԱ-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻〼ぁ-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-舘並-龎ﬀ-ﬆﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-𐀺𐀼𐀽𐀿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨𝞪-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+" ) . toAutomaton ( ) ) ; private final CharacterRunAutomaton runAutomaton ; private final boolean lowerCase ; private int state ; private final CharTermAttribute termAtt = addAttribute ( CharTermAttribute . class ) ; private final OffsetAttribute offsetAtt = addAttribute ( OffsetAttribute . class ) ; int off = 0 ; private static enum State { SETREADER , RESET , INCREMENT , INCREMENT_FALSE , END , CLOSE ; } private MockTokenizer . State streamState = MockTokenizer . State . CLOSE ; private boolean enableChecks = true ; public MockTokenizer ( AttributeFactory factory , Reader input , CharacterRunAutomaton runAutomaton , boolean lowerCase ) { } public MockTokenizer ( Reader input , CharacterRunAutomaton runAutomaton , boolean lowerCase ) { } @ Override public final boolean incrementToken ( ) throws IOException { assert ( ! ( enableChecks ) ) || ( ( ( streamState ) == ( MockTokenizer . State . RESET ) ) || ( ( streamState ) == ( MockTokenizer . State . INCREMENT ) ) ) : "incrementToken()<seq2seq4repair_space>called<seq2seq4repair_space>while<seq2seq4repair_space>in<seq2seq4repair_space>wrong<seq2seq4repair_space>state:<seq2seq4repair_space>" + ( streamState ) ; clearAttributes ( ) ; for ( ; ; ) { int startOffset = off ; int cp = readCodePoint ( ) ; if ( cp < 0 ) { break ; } else if ( isTokenChar ( cp ) ) { int endOffset ; do { char [ ] chars = Character . toChars ( normalize ( cp ) ) ; for ( int i = 0 ; i < ( chars . length ) ; i ++ ) termAtt . append ( chars [ i ] ) ; endOffset = off ; cp = readCodePoint ( ) ; } while ( ( cp >= 0 ) && ( isTokenChar ( cp ) ) ) ; <START_BUG> offsetAtt . setOffset ( startOffset , endOffset ) ; <END_BUG> streamState = MockTokenizer . State . INCREMENT ; return true ; } } streamState = MockTokenizer . State . INCREMENT_FALSE ; return false ; } protected int readCodePoint ( ) throws IOException { } protected boolean isTokenChar ( int c ) { } protected int normalize ( int c ) { } @ Override public void reset ( ) throws IOException { } @ Override public void close ( ) throws IOException { } @ Override public void reset ( Reader input ) throws IOException { } @ Override public void end ( ) throws IOException { } public void setEnableChecks ( boolean enableChecks ) { } }

Fix:
offsetAtt . setOffset ( correctOffset ( startOffset ) , correctOffset ( endOffset ) ) ;