Source:
<code>private void detectDuplicateInputIds( Radix radix, int numberOfCollisions, Collector collector, ProgressListener progress ) throws InterruptedException { Comparator duplicateComparator = new Comparator() { @Override public boolean lt( long left, long pivot ) { long leftEId = dataCache.get( left ); long pivotEId = dataCache.get( pivot ); if ( comparator.lt( leftEId, pivotEId ) ) { return true; } if ( leftEId == pivotEId ) { return left < pivot; } return false; } @Override public boolean ge( long right, long pivot ) { long rightEId = dataCache.get( right ); long pivotEId = dataCache.get( pivot ); if ( comparator.ge( rightEId, pivotEId ) ) { return rightEId != pivotEId || right > pivot; } return false; } @Override public long dataValue( long nodeId ) { return dataCache.get( nodeId ); } }; new ParallelSort( radix, collisionNodeIdCache, numberOfCollisions - 1, collisionTrackerCache, processorsForParallelWork, progress, duplicateComparator ).run(); long previousEid = 0; int previousGroupId = -1; SourceInformation source = new SourceInformation(); SameInputIdDetector detector = new SameInputIdDetector(); progress.started( "DEDUPLICATE" ); for ( int i = 0; i < numberOfCollisions; i++ ) { long collisionIndex = collisionTrackerCache.get( i ); long nodeId = collisionNodeIdCache.get( collisionIndex ); long eid = dataCache.get( nodeId ); int groupId = groupOf( nodeId ); boolean same = eid == previousEid && previousGroupId == groupId; if ( !same ) { detector.clear(); } Object inputId = collisionValues.get( safeCastLongToInt( collisionIndex ) ); <START> int detectorIndex = detector.add( inputId ); <END> if ( detectorIndex != -1 ) { collector.collectDuplicateNode( inputId, nodeId, groups[groupId].name() ); } previousEid = eid; previousGroupId = groupId; progress.add( 1 ); } progress.done(); }</code><technical_language>add method loops inputIdArray, calling in a loop collisions. a quadratic algorithm here. is a problem</technical_language>

Target:
private void detectDuplicateInputIds( Radix radix, int numberOfCollisions, Collector collector, ProgressListener progress ) throws InterruptedException { Comparator duplicateComparator = new Comparator() { @Override public boolean lt( long left, long pivot ) { long leftEId = dataCache.get( left ); long pivotEId = dataCache.get( pivot ); if ( comparator.lt( leftEId, pivotEId ) ) { return true; } if ( leftEId == pivotEId ) { return left < pivot; } return false; } @Override public boolean ge( long right, long pivot ) { long rightEId = dataCache.get( right ); long pivotEId = dataCache.get( pivot ); if ( comparator.ge( rightEId, pivotEId ) ) { return rightEId != pivotEId || right > pivot; } return false; } @Override public long dataValue( long nodeId ) { return dataCache.get( nodeId ); } }; new ParallelSort( radix, collisionNodeIdCache, numberOfCollisions - 1, collisionTrackerCache, processorsForParallelWork, progress, duplicateComparator ).run(); long previousEid = 0; int previousGroupId = -1; SameInputIdDetector detector = new SameInputIdDetector(); progress.started( "DEDUPLICATE" ); for ( int i = 0; i < numberOfCollisions; i++ ) { long collisionIndex = collisionTrackerCache.get( i ); long nodeId = collisionNodeIdCache.get( collisionIndex ); long eid = dataCache.get( nodeId ); int groupId = groupOf( nodeId ); boolean same = eid == previousEid && previousGroupId == groupId; if ( !same ) { detector.clear(); } Object inputId = collisionValues.get( safeCastLongToInt( collisionIndex ) ); int detectorIndex = detector.add( inputId ); if ( detectorIndex != -1 ) { collector.collectDuplicateNode( inputId, nodeId, groups.get( groupId ).name() ); } previousEid = eid; previousGroupId = groupId; progress.add( 1 ); } progress.done(); }