Program: 160238
Project: 160238
Buggy source:
/** 
 * Reproduce a severe issue where the pusher stops working because it's remoteRequestExecutor is full of tasks which are all blocked trying to add more tasks to the queue.
 */
public void testRetryQueueDeadlock() throws Exception {
  com.couchbase.lite.replicator.RemoteRequestRetry.RETRY_DELAY_MS=5;
  PersistentCookieJar cookieStore=database.getPersistentCookieStore();
  CouchbaseLiteHttpClientFactory factory=new CouchbaseLiteHttpClientFactory(cookieStore);
  MockDispatcher dispatcher=new MockDispatcher();
  MockWebServer server=MockHelper.getMockWebServer(dispatcher);
  dispatcher.setServerType(MockDispatcher.ServerType.SYNC_GW);
  try {
    MockResponse response=new MockResponse().setResponseCode(503);
    WrappedSmartMockResponse wrapped=new WrappedSmartMockResponse(response);
    wrapped.setDelayMs(5);
    wrapped.setSticky(true);
    dispatcher.enqueueResponse(MockHelper.PATH_REGEX_CHECKPOINT,wrapped);
    server.start();
    URL url=new URL(String.format(Locale.ENGLISH,"String_Node_Str",server.url("String_Node_Str").url(),"String_Node_Str"));
    Map<String,Object> requestBody=new HashMap<String,Object>();
    requestBody.put("String_Node_Str","String_Node_Str");
    Map<String,Object> requestHeaders=new HashMap<String,Object>();
    int numRequests=10;
    final CountDownLatch received503Error=new CountDownLatch(numRequests);
    RemoteRequestCompletion completionBlock=new RemoteRequestCompletion(){
      @Override public void onCompletion(      Response httpResponse,      Object result,      Throwable e){
        if (e instanceof RemoteRequestResponseException) {
          RemoteRequestResponseException htre=(RemoteRequestResponseException)e;
          if (htre.getCode() == 503) {
            received503Error.countDown();
          }
        }
      }
    }
;
    ScheduledExecutorService requestExecutorService=Executors.newScheduledThreadPool(5);
    ScheduledExecutorService workExecutorService=Executors.newSingleThreadScheduledExecutor();
    List<Future> requestFutures=new ArrayList<Future>();
    for (int i=0; i < numRequests; i++) {
      RemoteRequestRetry request=new RemoteRequestRetry(RemoteRequestRetry.RemoteRequestType.REMOTE_REQUEST,requestExecutorService,workExecutorService,factory,"String_Node_Str",url,true,true,requestBody,null,database,requestHeaders,completionBlock);
      Future future=request.submit();
      requestFutures.add(future);
    }
    for (    Future future : requestFutures)     future.get();
    boolean success=received503Error.await(120,TimeUnit.SECONDS);
    assertTrue(success);
    Utils.shutdownAndAwaitTermination(requestExecutorService);
    Utils.shutdownAndAwaitTermination(workExecutorService);
  }
  finally {
    assertTrue(MockHelper.shutdown(server,dispatcher));
  }
}

Fixed version (ground truth):
/** 
 * Reproduce a severe issue where the pusher stops working because it's remoteRequestExecutor is full of tasks which are all blocked trying to add more tasks to the queue.
 */
public void testRetryQueueDeadlock() throws Exception {
  int ORG_RETRY_DELAY_MS=com.couchbase.lite.replicator.RemoteRequestRetry.RETRY_DELAY_MS;
  try {
    com.couchbase.lite.replicator.RemoteRequestRetry.RETRY_DELAY_MS=5;
    PersistentCookieJar cookieStore=database.getPersistentCookieStore();
    CouchbaseLiteHttpClientFactory factory=new CouchbaseLiteHttpClientFactory(cookieStore);
    MockDispatcher dispatcher=new MockDispatcher();
    MockWebServer server=MockHelper.getMockWebServer(dispatcher);
    dispatcher.setServerType(MockDispatcher.ServerType.SYNC_GW);
    try {
      int numRequests=10;
      for (int i=0; i < numRequests * 5; i++) {
        MockResponse response=new MockResponse().setResponseCode(503);
        WrappedSmartMockResponse wrapped=new WrappedSmartMockResponse(response);
        wrapped.setDelayMs(5);
        dispatcher.enqueueResponse(MockHelper.PATH_REGEX_CHECKPOINT,wrapped);
      }
      server.start();
      URL url=new URL(String.format(Locale.ENGLISH,"String_Node_Str",server.url("String_Node_Str").url(),"String_Node_Str"));
      Map<String,Object> requestBody=new HashMap<String,Object>();
      requestBody.put("String_Node_Str","String_Node_Str");
      Map<String,Object> requestHeaders=new HashMap<String,Object>();
      final CountDownLatch received503Error=new CountDownLatch(numRequests);
      RemoteRequestCompletion completionBlock=new RemoteRequestCompletion(){
        @Override public void onCompletion(        Response httpResponse,        Object result,        Throwable e){
          if (e instanceof RemoteRequestResponseException) {
            RemoteRequestResponseException htre=(RemoteRequestResponseException)e;
            if (htre.getCode() == 503) {
              received503Error.countDown();
            }
          }
        }
      }
;
      ScheduledExecutorService requestExecutorService=Executors.newScheduledThreadPool(5);
      ScheduledExecutorService workExecutorService=Executors.newSingleThreadScheduledExecutor();
      List<Future> requestFutures=new ArrayList<Future>();
      for (int i=0; i < numRequests; i++) {
        RemoteRequestRetry request=new RemoteRequestRetry(RemoteRequestRetry.RemoteRequestType.REMOTE_REQUEST,requestExecutorService,workExecutorService,factory,"String_Node_Str",url,true,true,requestBody,null,database,requestHeaders,completionBlock);
        Future future=request.submit();
        requestFutures.add(future);
      }
      for (      Future future : requestFutures)       future.get();
      boolean success=received503Error.await(120,TimeUnit.SECONDS);
      assertTrue(success);
      Utils.shutdownAndAwaitTermination(requestExecutorService);
      Utils.shutdownAndAwaitTermination(workExecutorService);
    }
  finally {
      assertTrue(MockHelper.shutdown(server,dispatcher));
    }
  }
  finally {
    com.couchbase.lite.replicator.RemoteRequestRetry.RETRY_DELAY_MS=ORG_RETRY_DELAY_MS;
  }
}