Program: 8707
Project: 8707
Buggy source:
private void flushBatch(Batch<T> batch) throws IOException, InterruptedException {
  Map<Address,T> batchMap=batch.getMap();
  List<Address> successfulAddresses=Lists.newArrayListWithExpectedSize(batchMap.size());
  Map<byte[],byte[]> successfulRows=Maps.newTreeMap(UnsignedBytes.lexicographicalComparator());
  long nanoTimeBeforeBatch=System.nanoTime();
  try {
    if (commitType.equals(CommitType.INCREMENT) && configuration.batchSize > 1) {
      Iterable<List<Map.Entry<Address,T>>> partitions=Iterables.partition(batchMap.entrySet(),configuration.batchSize);
      int batchesPerFlush=0;
      Map<BoxedByteArray,T> increments=new HashMap<>();
      Map<BoxedByteArray,Address> backwards=new HashMap<>();
      for (      List<Map.Entry<Address,T>> entries : partitions) {
        batchesPerFlush++;
        final long nanoTimeBeforeWrite=System.nanoTime();
        for (        Map.Entry<Address,T> entry : entries) {
          final Address address=entry.getKey();
          final T op=entry.getValue();
          final BoxedByteArray rowKey=new BoxedByteArray(ArrayUtils.addAll(uniqueCubeName,address.toWriteKey(idService)));
          increments.put(rowKey,op);
          backwards.put(rowKey,address);
        }
        Map<BoxedByteArray,byte[]> successes=increment(increments);
        for (        Map.Entry<BoxedByteArray,byte[]> entry : successes.entrySet()) {
          successfulAddresses.add(backwards.get(entry.getKey()));
          successfulRows.put(entry.getKey().bytes,entry.getValue());
        }
        long writeDurationNanos=System.nanoTime() - nanoTimeBeforeWrite;
        batchWritesTimer.update(writeDurationNanos,TimeUnit.NANOSECONDS);
      }
      batchesPerFlushHisto.update(batchesPerFlush);
      if (successfulAddresses.size() < batchMap.size()) {
        int failures=batchMap.size() - successfulAddresses.size();
        incrementFailuresPerFlush.update(failures);
        throw new IOException(String.format("String_Node_Str",failures,batchMap.size()));
      }
    }
 else {
      for (      Map.Entry<Address,T> entry : batchMap.entrySet()) {
        final Address address=entry.getKey();
        final T op=entry.getValue();
        final byte[] rowKey=ArrayUtils.addAll(uniqueCubeName,address.toWriteKey(idService));
        final long nanoTimeBeforeWrite=System.nanoTime();
        byte[] dbBytes;
switch (commitType) {
case INCREMENT:
          long postIncr=increment(rowKey,op);
        dbBytes=Bytes.toBytes(postIncr);
      break;
case READ_COMBINE_CAS:
    dbBytes=readCombineCas(rowKey,op);
  break;
case OVERWRITE:
overwrite(rowKey,op);
dbBytes=op.serialize();
break;
default :
throw new RuntimeException("String_Node_Str" + commitType);
}
long writeDurationNanos=System.nanoTime() - nanoTimeBeforeWrite;
singleWriteTimer.update(writeDurationNanos,TimeUnit.NANOSECONDS);
if (log.isDebugEnabled()) {
log.debug("String_Node_Str" + Arrays.toString(uniqueCubeName) + "String_Node_Str"+ address);
}
successfulAddresses.add(address);
successfulRows.put(rowKey,dbBytes);
}
}
long batchDurationNanos=System.nanoTime() - nanoTimeBeforeBatch;
flushSuccessTimer.update(batchDurationNanos,TimeUnit.NANOSECONDS);
}
 catch (IOException ioe) {
log.warn("String_Node_Str",ioe);
for (Address address : successfulAddresses) {
batch.getMap().remove(address);
}
long batchDurationNanos=System.nanoTime() - nanoTimeBeforeBatch;
flushFailTimer.update(batchDurationNanos,TimeUnit.NANOSECONDS);
throw ioe;
}
 finally {
try {
onFlush.apply(successfulRows);
}
 catch (Exception e) {
log.error("String_Node_Str",e);
}
}
}

Fixed version (ground truth):
private void flushBatch(Batch<T> batch) throws IOException, InterruptedException {
  Map<Address,T> batchMap=batch.getMap();
  List<Address> successfulAddresses=Lists.newArrayListWithExpectedSize(batchMap.size());
  Map<byte[],byte[]> successfulRows=Maps.newTreeMap(UnsignedBytes.lexicographicalComparator());
  long nanoTimeBeforeBatch=System.nanoTime();
  try {
    if (commitType.equals(CommitType.INCREMENT) && configuration.batchSize > 1) {
      flushBatchedIncrement(batchMap,successfulAddresses,successfulRows);
    }
 else {
      for (      Map.Entry<Address,T> entry : batchMap.entrySet()) {
        final Address address=entry.getKey();
        final T op=entry.getValue();
        final byte[] rowKey=ArrayUtils.addAll(uniqueCubeName,address.toWriteKey(idService));
        final long nanoTimeBeforeWrite=System.nanoTime();
        byte[] dbBytes;
switch (commitType) {
case INCREMENT:
          long postIncr=increment(rowKey,op);
        dbBytes=Bytes.toBytes(postIncr);
      break;
case READ_COMBINE_CAS:
    dbBytes=readCombineCas(rowKey,op);
  break;
case OVERWRITE:
overwrite(rowKey,op);
dbBytes=op.serialize();
break;
default :
throw new RuntimeException("String_Node_Str" + commitType);
}
long writeDurationNanos=System.nanoTime() - nanoTimeBeforeWrite;
singleWriteTimer.update(writeDurationNanos,TimeUnit.NANOSECONDS);
if (log.isDebugEnabled()) {
log.debug("String_Node_Str" + Arrays.toString(uniqueCubeName) + "String_Node_Str"+ address);
}
successfulAddresses.add(address);
successfulRows.put(rowKey,dbBytes);
}
}
long batchDurationNanos=System.nanoTime() - nanoTimeBeforeBatch;
flushSuccessTimer.update(batchDurationNanos,TimeUnit.NANOSECONDS);
}
 catch (IOException ioe) {
log.warn("String_Node_Str",ioe);
if (ioe instanceof RetriesExhaustedWithDetailsException) {
RetriesExhaustedWithDetailsException details=(RetriesExhaustedWithDetailsException)ioe;
MDC.put("String_Node_Str",String.valueOf(details.mayHaveClusterIssues()));
for (int i=0; i < details.getNumExceptions(); ++i) {
MDC.put("String_Node_Str",details.getHostnamePort(i));
MDC.put("String_Node_Str",BaseEncoding.base64().encode(details.getRow(i).getRow()));
log.error(String.format("String_Node_Str",i),details.getCause(i));
}
MDC.clear();
}
for (Address address : successfulAddresses) {
batch.getMap().remove(address);
}
long batchDurationNanos=System.nanoTime() - nanoTimeBeforeBatch;
flushFailTimer.update(batchDurationNanos,TimeUnit.NANOSECONDS);
throw ioe;
}
 finally {
try {
onFlush.apply(successfulRows);
}
 catch (Exception e) {
log.error("String_Node_Str",e);
}
}
}