Buggy code:
final class DocInverterPerField extends DocFieldConsumerPerField { private final DocInverterPerThread perThread ; private final FieldInfo fieldInfo ; final InvertedDocConsumerPerField consumer ; final InvertedDocEndConsumerPerField endConsumer ; final DocState docState ; final FieldInvertState fieldState ; public DocInverterPerField ( DocInverterPerThread perThread , FieldInfo fieldInfo ) { } void abort ( ) { } public void processFields ( final Fieldable [ ] fields , final int count ) throws IOException { fieldState . reset ( docState . doc . getBoost ( ) ) ; final int maxFieldLength = docState . maxFieldLength ; final boolean doInvert = consumer . start ( fields , count ) ; for ( int i = 0 ; i < count ; i ++ ) { final Fieldable field = fields [ i ] ; if ( ( field . isIndexed ( ) ) && doInvert ) { if ( ( fieldState . length ) > 0 ) fieldState . position += docState . analyzer . getPositionIncrementGap ( fieldInfo . name ) ; if ( ! ( field . isTokenized ( ) ) ) { String stringValue = field . stringValue ( ) ; final int valueLength = stringValue . length ( ) ; <START_BUG> Token token = perThread . localToken . reinit ( stringValue , fieldState . offset , ( ( fieldState . offset ) + valueLength ) ) ; <END_BUG> boolean success = false ; try { consumer . add ( token ) ; success = true ; } finally { if ( ! success ) docState . docWriter . setAborting ( ) ; } fieldState . offset += valueLength ; ( fieldState . length ) ++ ; ( fieldState . position ) ++ ; } else { final TokenStream stream ; final TokenStream streamValue = field . tokenStreamValue ( ) ; if ( streamValue != null ) stream = streamValue ; else { final Reader reader ; final Reader readerValue = field . readerValue ( ) ; if ( readerValue != null ) reader = readerValue ; else { String stringValue = field . stringValue ( ) ; if ( stringValue == null ) throw new IllegalArgumentException ( "field<seq2seq4repair_space>must<seq2seq4repair_space>have<seq2seq4repair_space>either<seq2seq4repair_space>TokenStream,<seq2seq4repair_space>String<seq2seq4repair_space>or<seq2seq4repair_space>Reader<seq2seq4repair_space>value" ) ; perThread . stringReader . init ( stringValue ) ; reader = perThread . stringReader ; } stream = docState . analyzer . reusableTokenStream ( fieldInfo . name , reader ) ; } stream . reset ( ) ; try { int offsetEnd = ( fieldState . offset ) - 1 ; final Token localToken = perThread . localToken ; for ( ; ; ) { Token token = stream . next ( localToken ) ; if ( token == null ) break ; final int posIncr = token . getPositionIncrement ( ) ; fieldState . position += posIncr - 1 ; if ( posIncr == 0 ) ( fieldState . numOverlap ) ++ ; boolean success = false ; try { consumer . add ( token ) ; success = true ; } finally { if ( ! success ) docState . docWriter . setAborting ( ) ; } ( fieldState . position ) ++ ; offsetEnd = ( fieldState . offset ) + ( token . endOffset ( ) ) ; if ( ( ++ ( fieldState . length ) ) >= maxFieldLength ) { if ( ( docState . infoStream ) != null ) docState . infoStream . println ( ( ( ( ( "maxFieldLength<seq2seq4repair_space>" + maxFieldLength ) + "<seq2seq4repair_space>reached<seq2seq4repair_space>for<seq2seq4repair_space>field<seq2seq4repair_space>" ) + ( fieldInfo . name ) ) + ",<seq2seq4repair_space>ignoring<seq2seq4repair_space>following<seq2seq4repair_space>tokens" ) ) ; break ; } } fieldState . offset = offsetEnd + 1 ; } finally { stream . close ( ) ; } } fieldState . boost *= field . getBoost ( ) ; } } consumer . finish ( ) ; endConsumer . finish ( ) ; } }

Fix:
Token token = perThread . localToken . reinit ( stringValue , 0 , valueLength ) ;