Source:
<code><START> public Pair<String, List<LayoutToken>> processShort(List<LayoutToken> tokens, Document doc) { <END> SortedSet<DocumentPiece> documentParts = new TreeSet<DocumentPiece>(); List<List<LayoutToken>> tokenChunks = new ArrayList<List<LayoutToken>>(); List<LayoutToken> currentChunk = new ArrayList<LayoutToken>(); int currentPos = 0; for(LayoutToken token : tokens) { if (currentChunk.size() == 0) { currentChunk.add(token); currentPos = token.getOffset() + token.getText().length(); } else { int tokenPos = token.getOffset(); if (currentPos+1 == tokenPos) { currentChunk.add(token); currentPos = token.getOffset() + token.getText().length(); } else { tokenChunks.add(currentChunk); currentChunk = new ArrayList<LayoutToken>(); currentChunk.add(token); currentPos = token.getOffset() + token.getText().length(); } } } tokenChunks.add(currentChunk); for(List<LayoutToken> chunk : tokenChunks) { int endInd = chunk.size()-1; int posStartAbstract = getDocIndexToken(doc, chunk.get(0)); int posEndAbstract = getDocIndexToken(doc, chunk.get(endInd)); DocumentPointer dp1 = new DocumentPointer(doc, chunk.get(0).getBlockPtr(), posStartAbstract); DocumentPointer dp2 = new DocumentPointer(doc, chunk.get(endInd).getBlockPtr(), posEndAbstract); DocumentPiece piece = new DocumentPiece(dp1, dp2); documentParts.add(piece); } Pair<String, LayoutTokenization> featSeg = getBodyTextFeatured(doc, documentParts); String res = null; List<LayoutToken> layoutTokenization = null; if (featSeg != null) { String featuredText = featSeg.getLeft(); LayoutTokenization layouts = featSeg.getRight(); if (layouts != null) layoutTokenization = layouts.getTokenization(); if ( (featuredText != null) && (featuredText.trim().length() > 0) ) { res = label(featuredText); } } return Pair.of(res, layoutTokenization); }</code><technical_language>I (I investigating another problem) ended this problem, currentPos + textLenght == nextToken.getOffset()</technical_language>

Target:
public Pair<String, List<LayoutToken>> processShort(List<LayoutToken> tokens, Document doc) { if (CollectionUtils.isEmpty(tokens)) return null; SortedSet<DocumentPiece> documentParts = new TreeSet<>(); List<List<LayoutToken>> tokenChunks = new ArrayList<>(); List<LayoutToken> currentChunk = new ArrayList<>(); int currentPos = 0; for(LayoutToken token : tokens) { if (currentChunk.size() != 0) { int tokenPos = token.getOffset(); if (currentPos != tokenPos) { tokenChunks.add(currentChunk); currentChunk = new ArrayList<LayoutToken>(); } } currentChunk.add(token); currentPos = token.getOffset() + token.getText().length(); } tokenChunks.add(currentChunk); for(List<LayoutToken> chunk : tokenChunks) { int endInd = chunk.size()-1; int posStartAbstract = getDocIndexToken(doc, chunk.get(0)); int posEndAbstract = getDocIndexToken(doc, chunk.get(endInd)); DocumentPointer dp1 = new DocumentPointer(doc, chunk.get(0).getBlockPtr(), posStartAbstract); DocumentPointer dp2 = new DocumentPointer(doc, chunk.get(endInd).getBlockPtr(), posEndAbstract); DocumentPiece piece = new DocumentPiece(dp1, dp2); documentParts.add(piece); } Pair<String, LayoutTokenization> featSeg = getBodyTextFeatured(doc, documentParts); String res = null; List<LayoutToken> layoutTokenization = null; if (featSeg != null) { String featuredText = featSeg.getLeft(); LayoutTokenization layouts = featSeg.getRight(); if (layouts != null) layoutTokenization = layouts.getTokenization(); if ( (featuredText != null) && (featuredText.trim().length() > 0) ) { res = label(featuredText); } } return Pair.of(res, layoutTokenization); }