Program: 80300
Project: 80300
Buggy source:
/** 
 * Update master computable result to master znode. At the same time clean znodes for old iterations. Iteration 0 and last iteration will not be removed for fail over.
 */
@Override public void postIteration(final MasterContext<MASTER_RESULT,WORKER_RESULT> context){
  context.setWorkerResults(null);
  new BasicCoordinatorCommand(){
    @Override public void doExecute() throws KeeperException, InterruptedException {
      NettyMasterCoordinator.this.masterResult=context.getMasterResult();
      updateMasterHaltStatus(context);
      boolean isSplit=false;
      String appCurrentMasterNode=getCurrentMasterNode(context.getAppId(),context.getCurrentIteration()).toString();
      String appCurrentMasterSplitNode=getCurrentMasterSplitNode(context.getAppId(),context.getCurrentIteration()).toString();
      LOG.debug("String_Node_Str",context.getMasterResult());
      final long start=System.nanoTime();
      try {
        byte[] bytes=getMasterSerializer().objectToBytes(context.getMasterResult());
        isSplit=setBytesToZNode(appCurrentMasterNode,appCurrentMasterSplitNode,bytes,CreateMode.PERSISTENT);
synchronized (LOCK) {
          clear(context.getProps());
          NettyMasterCoordinator.this.currentInteration=context.getCurrentIteration() + 1;
          NettyMasterCoordinator.this.canUpdateWorkerResultMap=true;
        }
      }
 catch (      KeeperException.NodeExistsException e) {
        LOG.warn("String_Node_Str",e);
      }
      LOG.info("String_Node_Str",TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
      final int resultCleanUpInterval=NumberFormatUtils.getInt(context.getProps().getProperty(GuaguaConstants.GUAGUA_CLEANUP_INTERVAL),GuaguaConstants.GUAGUA_DEFAULT_CLEANUP_INTERVAL);
      if (context.getCurrentIteration() >= (resultCleanUpInterval + 1)) {
        final boolean isLocalSplit=isSplit;
        NettyMasterCoordinator.this.cleanOldZkDataThreadPool.submit(new Runnable(){
          @Override public void run(){
            String znode=getMasterNode(context.getAppId(),context.getCurrentIteration() - resultCleanUpInterval).toString();
            try {
              getZooKeeper().deleteExt(znode,-1,false);
              if (isLocalSplit) {
                znode=getCurrentMasterSplitNode(context.getAppId(),context.getCurrentIteration() - resultCleanUpInterval).toString();
                getZooKeeper().deleteExt(znode,-1,true);
              }
            }
 catch (            KeeperException.NoNodeException e) {
              if (System.nanoTime() % 20 == 0) {
                LOG.warn("String_Node_Str",znode);
              }
            }
catch (            KeeperException ignore) {
            }
catch (            InterruptedException ignore) {
            }
          }
        }
);
      }
      LOG.info("String_Node_Str");
    }
  }
.execute();
}

Fixed version (ground truth):
/** 
 * Update master computable result to master znode. At the same time clean znodes for old iterations. Iteration 0 and last iteration will not be removed for fail over.
 */
@Override public void postIteration(final MasterContext<MASTER_RESULT,WORKER_RESULT> context){
  context.setWorkerResults(null);
  new BasicCoordinatorCommand(){
    @Override public void doExecute() throws KeeperException, InterruptedException {
      NettyMasterCoordinator.this.masterResult=context.getMasterResult();
      updateMasterHaltStatus(context);
      boolean isSplit=false;
      String appCurrentMasterNode=getCurrentMasterNode(context.getAppId(),context.getCurrentIteration()).toString();
      String appCurrentMasterSplitNode=getCurrentMasterSplitNode(context.getAppId(),context.getCurrentIteration()).toString();
      LOG.debug("String_Node_Str",context.getMasterResult());
      final long start=System.nanoTime();
      try {
        byte[] bytes=getMasterSerializer().objectToBytes(context.getMasterResult());
        isSplit=setBytesToZNode(appCurrentMasterNode,appCurrentMasterSplitNode,bytes,CreateMode.PERSISTENT);
synchronized (LOCK) {
          clear(context.getProps());
          NettyMasterCoordinator.this.currentInteration=context.getCurrentIteration() + 1;
          NettyMasterCoordinator.this.canUpdateWorkerResultMap=true;
        }
      }
 catch (      KeeperException.NodeExistsException e) {
        LOG.warn("String_Node_Str",e);
      }
      LOG.info("String_Node_Str",TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
      final int resultCleanUpInterval=NumberFormatUtils.getInt(context.getProps().getProperty(GuaguaConstants.GUAGUA_CLEANUP_INTERVAL),GuaguaConstants.GUAGUA_DEFAULT_CLEANUP_INTERVAL);
      if (context.getCurrentIteration() >= (resultCleanUpInterval + 1)) {
        final boolean isLocalSplit=isSplit;
        NettyMasterCoordinator.this.cleanOldZkDataThreadPool.submit(new Runnable(){
          @Override public void run(){
            String znode=getMasterNode(context.getAppId(),context.getCurrentIteration() - resultCleanUpInterval).toString();
            try {
              getZooKeeper().deleteExt(znode,-1,false);
              if (isLocalSplit) {
                znode=getCurrentMasterSplitNode(context.getAppId(),context.getCurrentIteration() - resultCleanUpInterval).toString();
                getZooKeeper().deleteExt(znode,-1,true);
              }
            }
 catch (            KeeperException.NoNodeException e) {
              if (System.nanoTime() % 20 == 0) {
                LOG.warn("String_Node_Str",znode);
              }
            }
catch (            KeeperException ignore) {
            }
catch (            InterruptedException ignore) {
            }
          }
        }
);
      }
      LOG.debug("String_Node_Str");
    }
  }
.execute();
}