Program: 13125
Project: 13125
Buggy source:
@Test public void testPrimaryRelocationWhileIndexing() throws Exception {
  final int numberOfRelocations=1;
  final int numberOfWriters=2;
  final String node1=internalCluster().startNode();
  BlobAdminClient blobAdminClient=internalCluster().getInstance(BlobAdminClient.class,node1);
  logger.trace("String_Node_Str");
  Settings indexSettings=Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS,0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS,1).build();
  blobAdminClient.createBlobTable("String_Node_Str",indexSettings).get();
  logger.trace("String_Node_Str");
  final String node2=internalCluster().startNode();
  ensureGreen();
  final AtomicLong idGenerator=new AtomicLong();
  final AtomicLong indexCounter=new AtomicLong();
  final AtomicBoolean stop=new AtomicBoolean(false);
  Thread[] writers=new Thread[numberOfWriters];
  final CountDownLatch stopLatch=new CountDownLatch(writers.length);
  logger.trace("String_Node_Str",writers.length);
  final List<String> uploadedDigests=Collections.synchronizedList(new ArrayList<String>(writers.length));
  for (int i=0; i < writers.length; i++) {
    final int indexerId=i;
    writers[i]=new Thread(){
      @Override public void run(){
        try {
          logger.trace("String_Node_Str",indexerId);
          while (!stop.get()) {
            long id=idGenerator.incrementAndGet();
            String digest=uploadFile(internalCluster().client(node1),genFile(id));
            uploadedDigests.add(digest);
            indexCounter.incrementAndGet();
          }
          logger.trace("String_Node_Str",indexerId);
        }
 catch (        Exception e) {
          logger.warn("String_Node_Str",e,indexerId);
        }
 finally {
          stopLatch.countDown();
        }
      }
    }
;
    writers[i].setName("String_Node_Str");
    writers[i].setDaemon(true);
    writers[i].start();
  }
  logger.trace("String_Node_Str");
  while (uploadedDigests.size() < 2) {
    Thread.sleep(10);
  }
  logger.trace("String_Node_Str");
  timeBetweenChunks.set(10);
  logger.trace("String_Node_Str");
  for (int i=0; i < numberOfRelocations; i++) {
    String fromNode=(i % 2 == 0) ? node1 : node2;
    String toNode=node1.equals(fromNode) ? node2 : node1;
    logger.trace("String_Node_Str",fromNode,toNode);
    internalCluster().client(node1).admin().cluster().prepareReroute().add(new MoveAllocationCommand(new ShardId(BlobIndex.fullIndexName("String_Node_Str"),0),fromNode,toNode)).execute().actionGet();
    ClusterHealthResponse clusterHealthResponse=internalCluster().client(node1).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForRelocatingShards(0).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(),equalTo(false));
    clusterHealthResponse=internalCluster().client(node2).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForRelocatingShards(0).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(),equalTo(false));
    logger.trace("String_Node_Str",fromNode,toNode);
  }
  logger.trace("String_Node_Str");
  logger.trace("String_Node_Str");
  timeBetweenChunks.set(0);
  stop.set(true);
  stopLatch.await(60,TimeUnit.SECONDS);
  logger.trace("String_Node_Str");
  logger.trace("String_Node_Str",indexCounter.get(),uploadedDigests.size());
  assertEquals(indexCounter.get(),uploadedDigests.size());
  BlobIndicesService blobIndicesService=internalCluster().getInstance(BlobIndicesService.class,node2);
  for (  String digest : uploadedDigests) {
    BlobShard blobShard=blobIndicesService.localBlobShard(BlobIndex.fullIndexName("String_Node_Str"),digest);
    long length=blobShard.blobContainer().getFile(digest).length();
    assertThat(length,greaterThanOrEqualTo(1L));
  }
  for (  Thread writer : writers) {
    writer.join(6000);
  }
}

Fixed version (ground truth):
@Test public void testPrimaryRelocationWhileIndexing() throws Exception {
  final int numberOfRelocations=1;
  final int numberOfWriters=2;
  final String node1=internalCluster().startNode();
  BlobAdminClient blobAdminClient=internalCluster().getInstance(BlobAdminClient.class,node1);
  logger.trace("String_Node_Str");
  Settings indexSettings=Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS,0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS,1).build();
  blobAdminClient.createBlobTable("String_Node_Str",indexSettings).get();
  logger.trace("String_Node_Str");
  final String node2=internalCluster().startNode();
  ensureGreen();
  final AtomicLong idGenerator=new AtomicLong();
  final AtomicLong indexCounter=new AtomicLong();
  final AtomicBoolean stop=new AtomicBoolean(false);
  Thread[] writers=new Thread[numberOfWriters];
  final CountDownLatch stopLatch=new CountDownLatch(writers.length);
  logger.trace("String_Node_Str",writers.length);
  final List<String> uploadedDigests=Collections.synchronizedList(new ArrayList<String>(writers.length));
  for (int i=0; i < writers.length; i++) {
    final int indexerId=i;
    writers[i]=new Thread(){
      @Override public void run(){
        try {
          logger.trace("String_Node_Str",indexerId);
          while (!stop.get()) {
            long id=idGenerator.incrementAndGet();
            String digest=uploadFile(internalCluster().client(node1),genFile(id));
            uploadedDigests.add(digest);
            indexCounter.incrementAndGet();
          }
          logger.trace("String_Node_Str",indexerId);
        }
 catch (        Exception e) {
          logger.warn("String_Node_Str",e,indexerId);
        }
 finally {
          stopLatch.countDown();
        }
      }
    }
;
    writers[i].setName("String_Node_Str");
    writers[i].setDaemon(true);
    writers[i].start();
  }
  logger.trace("String_Node_Str");
  while (uploadedDigests.size() < 2) {
    Thread.sleep(10);
  }
  logger.trace("String_Node_Str");
  timeBetweenChunks.set(10);
  logger.trace("String_Node_Str");
  for (int i=0; i < numberOfRelocations; i++) {
    String fromNode=(i % 2 == 0) ? node1 : node2;
    String toNode=node1.equals(fromNode) ? node2 : node1;
    logger.trace("String_Node_Str",fromNode,toNode);
    internalCluster().client(node1).admin().cluster().prepareReroute().add(new MoveAllocationCommand(new ShardId(BlobIndex.fullIndexName("String_Node_Str"),0),fromNode,toNode)).execute().actionGet();
    ClusterHealthResponse clusterHealthResponse=internalCluster().client(node1).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForRelocatingShards(0).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(),equalTo(false));
    clusterHealthResponse=internalCluster().client(node2).admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForRelocatingShards(0).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(),equalTo(false));
    logger.trace("String_Node_Str",fromNode,toNode);
  }
  logger.trace("String_Node_Str");
  logger.trace("String_Node_Str");
  timeBetweenChunks.set(0);
  stop.set(true);
  assertThat(stopLatch.await(60,TimeUnit.SECONDS),is(true));
  logger.trace("String_Node_Str");
  logger.trace("String_Node_Str",indexCounter.get(),uploadedDigests.size());
  assertEquals(indexCounter.get(),uploadedDigests.size());
  BlobIndicesService blobIndicesService=internalCluster().getInstance(BlobIndicesService.class,node2);
  for (  String digest : uploadedDigests) {
    BlobShard blobShard=blobIndicesService.localBlobShard(BlobIndex.fullIndexName("String_Node_Str"),digest);
    long length=blobShard.blobContainer().getFile(digest).length();
    assertThat(length,greaterThanOrEqualTo(1L));
  }
  for (  Thread writer : writers) {
    writer.join(6000);
  }
}