Repository: prestodb/presto
Issue: Deadlock in task info fetching

Hit this one in production. Suddenly all queries get queued and all clients start timing out. [Here](https://gist.githubusercontent.com/nezihyigitbasi/108b844625d69de66bd8d74e427ea876/raw/c22dc663517fa991b87c162de2b387c0e8af0d69/presto-stack) is the full stack trace.

```
Found one Java-level deadlock:
=============================
"http-worker-2010473":
  waiting to lock monitor 0x00007f8fed4a6fb8 (object 0x00007f9615a7e020, a com.facebook.presto.execution.resourceGroups.ResourceGroup$RootResourceGroup),
  which is held by "ResourceGroupManager"
"ResourceGroupManager":
  waiting to lock monitor 0x00007f91f0cc3598 (object 0x00007f9a9f000a48, a com.facebook.presto.execution.SqlStageExecution),
  which is held by "HttpRemoteTask-20160926_222904_08073_w6q27.1.105-2010288"
"HttpRemoteTask-20160926_222904_08073_w6q27.1.105-2010288":
  waiting to lock monitor 0x00007f90ed8c3ee8 (object 0x00007f9a9f000870, a com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher),
  which is held by "ContinuousTaskStatusFetcher-20160926_222904_08073_w6q27.1.105-2009564"
"ContinuousTaskStatusFetcher-20160926_222904_08073_w6q27.1.105-2009564":
  waiting to lock monitor 0x00007f910c0b5948 (object 0x00007f9a9f0008e0, a com.facebook.presto.server.remotetask.HttpRemoteTask),
  which is held by "HttpRemoteTask-20160926_222904_08073_w6q27.1.105-2010288"

Java stack information for the threads listed above:
===================================================
"http-worker-2010473":
    at com.facebook.presto.execution.resourceGroups.ResourceGroup.run(ResourceGroup.java:322)
    - waiting to lock <0x00007f9615a7e020> (a com.facebook.presto.execution.resourceGroups.ResourceGroup$RootResourceGroup)
    at com.facebook.presto.execution.resourceGroups.ResourceGroupManager.submit(ResourceGroupManager.java:89)
    at com.facebook.presto.execution.SqlQueryManager.createQuery(SqlQueryManager.java:348)
    at com.facebook.presto.server.StatementResource$Query.<init>(StatementResource.java:308)
    at com.facebook.presto.server.StatementResource.createQuery(StatementResource.java:171)
    at sun.reflect.GeneratedMethodAccessor1181.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)
    at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144)
    at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161)
    at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:160)
    at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99)
    at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389)
    at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347)
    at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102)
    at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326)
    at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
    at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
    at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
    at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
    at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
    at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)
    at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)
    at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)
    at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:473)
    at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:427)
    at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:388)
    at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:341)
    at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:228)
    at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:845)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1689)
    at io.airlift.http.server.TraceTokenFilter.doFilter(TraceTokenFilter.java:63)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
    at io.airlift.http.server.TimingFilter.doFilter(TimingFilter.java:52)
    at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
    at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
    at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:396)
    at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1176)
    at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
    at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1106)
    at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
    at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
    at org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:169)
    at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
    at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
    at org.eclipse.jetty.server.Server.handle(Server.java:518)
    at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:314)
    at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:253)
    at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
    at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
    at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
    at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
    at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
    at java.lang.Thread.run(Thread.java:745)
"ResourceGroupManager":
    at com.facebook.presto.execution.SqlStageExecution.getMemoryReservation(SqlStageExecution.java:184)
    - waiting to lock <0x00007f9a9f000a48> (a com.facebook.presto.execution.SqlStageExecution)
    at com.facebook.presto.execution.scheduler.SqlQueryScheduler$$Lambda$792/605126606.applyAsLong(Unknown Source)
    at java.util.stream.ReferencePipeline$5$1.accept(ReferencePipeline.java:227)
    at java.util.Iterator.forEachRemaining(Iterator.java:116)
    at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
    at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
    at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
    at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
    at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
    at java.util.stream.LongPipeline.reduce(LongPipeline.java:438)
    at java.util.stream.LongPipeline.sum(LongPipeline.java:396)
    at com.facebook.presto.execution.scheduler.SqlQueryScheduler.getTotalMemoryReservation(SqlQueryScheduler.java:310)
    at com.facebook.presto.execution.SqlQueryExecution.getTotalMemoryReservation(SqlQueryExecution.java:188)
    at com.facebook.presto.execution.resourceGroups.ResourceGroup.internalRefreshStats(ResourceGroup.java:438)
    - locked <0x00007f9615a7e020> (a com.facebook.presto.execution.resourceGroups.ResourceGroup$RootResourceGroup)
    at com.facebook.presto.execution.resourceGroups.ResourceGroup.internalRefreshStats(ResourceGroup.java:445)
    - locked <0x00007f9615a7e020> (a com.facebook.presto.execution.resourceGroups.ResourceGroup$RootResourceGroup)
    at com.facebook.presto.execution.resourceGroups.ResourceGroup$RootResourceGroup.processQueuedQueries(ResourceGroup.java:580)
    - locked <0x00007f9615a7e020> (a com.facebook.presto.execution.resourceGroups.ResourceGroup$RootResourceGroup)
    at com.facebook.presto.execution.resourceGroups.ResourceGroupManager.refreshAndStartQueries(ResourceGroupManager.java:110)
    at com.facebook.presto.execution.resourceGroups.ResourceGroupManager$$Lambda$140/724736957.run(Unknown Source)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
"HttpRemoteTask-20160926_222904_08073_w6q27.1.105-2010288":
    at com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher.updateTaskStatus(ContinuousTaskStatusFetcher.java:214)
    - waiting to lock <0x00007f9a9f000870> (a com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher)
    at com.facebook.presto.server.remotetask.HttpRemoteTask.abort(HttpRemoteTask.java:560)
    - locked <0x00007f9a9f0008e0> (a com.facebook.presto.server.remotetask.HttpRemoteTask)
    at com.facebook.presto.server.remotetask.HttpRemoteTask.abort(HttpRemoteTask.java:552)
    - locked <0x00007f9a9f0008e0> (a com.facebook.presto.server.remotetask.HttpRemoteTask)
    at com.facebook.presto.execution.SqlStageExecution$$Lambda$1037/616866428.accept(Unknown Source)
    at java.lang.Iterable.forEach(Iterable.java:75)
    at com.facebook.presto.execution.SqlStageExecution.abort(SqlStageExecution.java:179)
    - locked <0x00007f9a9f000a48> (a com.facebook.presto.execution.SqlStageExecution)
    at com.facebook.presto.execution.scheduler.SqlQueryScheduler$$Lambda$1035/234080167.accept(Unknown Source)
    at java.util.Iterator.forEachRemaining(Iterator.java:116)
    at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
    at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580)
    at com.facebook.presto.execution.scheduler.SqlQueryScheduler.abort(SqlQueryScheduler.java:412)
    at com.facebook.presto.execution.SqlQueryExecution.lambda$new$0(SqlQueryExecution.java:154)
    at com.facebook.presto.execution.SqlQueryExecution$$Lambda$440/1153217709.stateChanged(Unknown Source)
    at com.facebook.presto.execution.StateMachine.lambda$fireStateChanged$0(StateMachine.java:225)
    at com.facebook.presto.execution.StateMachine$$Lambda$413/743619379.run(Unknown Source)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
"ContinuousTaskStatusFetcher-20160926_222904_08073_w6q27.1.105-2009564":
    at com.facebook.presto.server.remotetask.HttpRemoteTask.abort(HttpRemoteTask.java:557)
    - waiting to lock <0x00007f9a9f0008e0> (a com.facebook.presto.server.remotetask.HttpRemoteTask)
    at com.facebook.presto.server.remotetask.HttpRemoteTask.failTask(HttpRemoteTask.java:621)
    at com.facebook.presto.server.remotetask.HttpRemoteTask$$Lambda$817/1647878013.accept(Unknown Source)
    at com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher.updateTaskStatus(ContinuousTaskStatusFetcher.java:234)
    - locked <0x00007f9a9f000870> (a com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher)
    at com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher.success(ContinuousTaskStatusFetcher.java:168)
    at com.facebook.presto.server.remotetask.ContinuousTaskStatusFetcher.success(ContinuousTaskStatusFetcher.java:52)
    at com.facebook.presto.server.remotetask.SimpleHttpResponseHandler.onSuccess(SimpleHttpResponseHandler.java:49)
    at com.facebook.presto.server.remotetask.SimpleHttpResponseHandler.onSuccess(SimpleHttpResponseHandler.java:27)
    at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319)
    at io.airlift.concurrent.BoundedExecutor.drainQueue(BoundedExecutor.java:77)
    at io.airlift.concurrent.BoundedExecutor$$Lambda$437/311179219.run(Unknown Source)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)

Found 1 deadlock.
```


Changed files:
presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java

diff --git a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java
index 77d8dacfeb..12bedf066c 100644
--- a/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java
+++ b/presto-main/src/main/java/com/facebook/presto/server/remotetask/ContinuousTaskStatusFetcher.java
@@ -208,7 +208,7 @@ class ContinuousTaskStatusFetcher
         }
     }
 
-    synchronized void updateTaskStatus(TaskStatus newValue)
+    void updateTaskStatus(TaskStatus newValue)
     {
         // change to new value if old value is not changed and new value has a newer version
         AtomicBoolean taskMismatch = new AtomicBoolean();