Repository: dask/distributed

Issue:
Client spews errors in JupyterLab during `compute`
In recent versions of distributed, during a compute, tons of errors sometimes start spewing out in JupyterLab like this:

![Screen Shot 2021-10-27 at 3 53 08 PM](https://user-images.githubusercontent.com/3309802/139152831-2fe9f2be-0ab5-4600-a83a-51732429d620.png)

I've heard other people complain about this too.

For searchability, here are some of the logs:
<details>

```
tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x1334d13d0>>, <Task finished name='Task-337' coro=<Cluster._sync_cluster_info() done, defined at /Users/gabe/dev/distributed/distributed/deploy/cluster.py:104> exception=OSError('Timed out trying to connect to tls://54.212.201.147:8786 after 5 s')>)
Traceback (most recent call last):
  File "/Users/gabe/dev/distributed/distributed/comm/tcp.py", line 398, in connect
    stream = await self.client.connect(
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/tcpclient.py", line 288, in connect
    stream = await stream.start_tls(
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/gabe/.pyenv/versions/3.9.1/lib/python3.9/asyncio/tasks.py", line 489, in wait_for
    fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/gabe/dev/distributed/distributed/comm/core.py", line 284, in connect
    comm = await asyncio.wait_for(
  File "/Users/gabe/.pyenv/versions/3.9.1/lib/python3.9/asyncio/tasks.py", line 491, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/ioloop.py", line 741, in _run_callback
    ret = callback()
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/ioloop.py", line 765, in _discard_future_result
    future.result()
  File "/Users/gabe/dev/distributed/distributed/deploy/cluster.py", line 105, in _sync_cluster_info
    await self.scheduler_comm.set_metadata(
  File "/Users/gabe/dev/distributed/distributed/core.py", line 785, in send_recv_from_rpc
    comm = await self.live_comm()
  File "/Users/gabe/dev/distributed/distributed/core.py", line 742, in live_comm
    comm = await connect(
  File "/Users/gabe/dev/distributed/distributed/comm/core.py", line 308, in connect
    raise OSError(
OSError: Timed out trying to connect to tls://54.212.201.147:8786 after 5 s
tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x1334d13d0>>, <Task finished name='Task-340' coro=<Cluster._sync_cluster_info() done, defined at /Users/gabe/dev/distributed/distributed/deploy/cluster.py:104> exception=OSError('Timed out trying to connect to tls://54.212.201.147:8786 after 5 s')>)
Traceback (most recent call last):
  File "/Users/gabe/dev/distributed/distributed/comm/tcp.py", line 398, in connect
    stream = await self.client.connect(
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/tcpclient.py", line 288, in connect
    stream = await stream.start_tls(
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/gabe/.pyenv/versions/3.9.1/lib/python3.9/asyncio/tasks.py", line 489, in wait_for
    fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/gabe/dev/distributed/distributed/comm/core.py", line 284, in connect
    comm = await asyncio.wait_for(
  File "/Users/gabe/.pyenv/versions/3.9.1/lib/python3.9/asyncio/tasks.py", line 491, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/ioloop.py", line 741, in _run_callback
    ret = callback()
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/ioloop.py", line 765, in _discard_future_result
    future.result()
  File "/Users/gabe/dev/distributed/distributed/deploy/cluster.py", line 105, in _sync_cluster_info
    await self.scheduler_comm.set_metadata(
  File "/Users/gabe/dev/distributed/distributed/core.py", line 785, in send_recv_from_rpc
    comm = await self.live_comm()
  File "/Users/gabe/dev/distributed/distributed/core.py", line 742, in live_comm
    comm = await connect(
  File "/Users/gabe/dev/distributed/distributed/comm/core.py", line 308, in connect
    raise OSError(
OSError: Timed out trying to connect to tls://54.212.201.147:8786 after 5 s
tornado.application - ERROR - Exception in callback functools.partial(<bound method IOLoop._discard_future_result of <zmq.eventloop.ioloop.ZMQIOLoop object at 0x1334d13d0>>, <Task finished name='Task-349' coro=<Cluster._sync_cluster_info() done, defined at /Users/gabe/dev/distributed/distributed/deploy/cluster.py:104> exception=OSError('Timed out trying to connect to tls://54.212.201.147:8786 after 5 s')>)
Traceback (most recent call last):
  File "/Users/gabe/dev/distributed/distributed/comm/tcp.py", line 398, in connect
    stream = await self.client.connect(
  File "/Users/gabe/dev/dask-playground/env/lib/python3.9/site-packages/tornado/tcpclient.py", line 288, in connect
    stream = await stream.start_tls(
asyncio.exceptions.CancelledError
```

</details>

From reading the traceback, it appears to be unhandled exceptions in the new cluster<->scheduler synced dict from https://github.com/dask/distributed/pull/5033. I'd guess that when the scheduler gets overwhelmed from any of the myriad of things that can block its event loop, something breaks. I'm not sure why it stays broken and the comm doesn't reconnect, but after one failure it seems you'll get this message once per second.
1. There should be error handling in the cluster info syncing; at a minimum, any error here should be handled and logged, but not allowed to propagate up
2. Why is this error happening, and why does it not seem to recover?

cc @jacobtomlinson 


Discussion:
Thanks for reporting @gjoseph92. Do you have a code example other can use to reproduce those errors? 
Is this specific to JupyterLab, or does the same thing happen in an IPython terminal or regular Python session?
@jrbourbeau I've seen this happening too, but the same code that spills the error, then it runs fine, and so far I couldn't narrow down a code example to reproduce. 

@ian-r-rose I saw it on pure notebooks but I'm assuming this is similar to JupyterLab. 
We've been seeing this `OSError: Timed out` a lot from the `_sync_cluster_info` periodic callback when Coiled cluster is (i) slow to start, (ii) slow to stopped or stopped without client knowing, or (iii) just too busy to respond.

One easy way to reproduce is to kill scheduler without the python client knowing.

I'm not familiar with what the sync is needed for, but I agree that it makes sense to handle the timeouts when they occur there. Simplest option is to silently catch, maybe like this: https://github.com/dask/distributed/compare/main...ntabris:main

Another option would be to add optional callback to `rpc` class, and if this callback is given, then call it when there's timeout rather than throwing `OSError`. This would allow user (e.g., `coiled`) to decide what to do with timeout... for instance, check cluster health if there's another way to do that (as there is for `coiled`).
> @ian-r-rose I saw it on pure notebooks but I'm assuming this is similar to JupyterLab.

Right, my question is: does this issue have anything to do with which Python interface people are using (pure interpreter, notebook, whatever)? It seems like the answer is no.
> One easy way to reproduce is to kill scheduler without the python client knowing

Thanks for pointing that out @ntabris. Here's a concrete snippet which one can use to trigger the errors (note that you'll need to wait a few seconds before the errors start appearing):

```python
from distributed import LocalCluster, Client

cluster = LocalCluster()
client = Client(cluster)

async def foo(dask_scheduler):
    await dask_scheduler.close()
    
client.run_on_scheduler(foo)
```

> does this issue have anything to do with which Python interface people are using (pure interpreter, notebook, whatever)? It seems like the answer is no.

I can confirm this happens in JupyterLab, IPython, and a plain Python session

FWIW @jcrist mentioned he's looking into this issue 
Thanks for finding a reproducer @jrbourbeau! Very helpful.

Resolving patch:
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,5 +1,4 @@
 import asyncio
-import copy
 import datetime
 import logging
 import threading
@@ -60,6 +59,7 @@ def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval
         self._sync_interval = parse_timedelta(
             scheduler_sync_interval, default="seconds"
         )
+        self._sync_cluster_info_task = None
 
         if name is None:
             name = str(uuid.uuid4())[:8]
@@ -94,23 +94,56 @@ async def _start(self):
         )
         self._cluster_info.update(info)
 
-        self.periodic_callbacks["sync-cluster-info"] = PeriodicCallback(
-            self._sync_cluster_info, self._sync_interval * 1000
-        )
+        # Start a background task for syncing cluster info with the scheduler
+        self._sync_cluster_info_task = asyncio.ensure_future(self._sync_cluster_info())
+
         for pc in self.periodic_callbacks.values():
             pc.start()
         self.status = Status.running
 
     async def _sync_cluster_info(self):
-        await self.scheduler_comm.set_metadata(
-            keys=["cluster-manager-info"],
-            value=copy.copy(self._cluster_info),
-        )
+        err_count = 0
+        warn_at = 5
+        max_interval = 10 * self._sync_interval
+        # Loop until the cluster is shutting down. We shouldn't really need
+        # this check (the `CancelledError` should be enough), but something
+        # deep in the comms code is silencing `CancelledError`s _some_ of the
+        # time, resulting in a cancellation not always bubbling back up to
+        # here. Relying on the status is fine though, not worth changing.
+        while self.status == Status.running:
+            try:
+                await self.scheduler_comm.set_metadata(
+                    keys=["cluster-manager-info"],
+                    value=self._cluster_info.copy(),
+                )
+                err_count = 0
+            except asyncio.CancelledError:
+                # Task is being closed. When we drop Python < 3.8 we can drop
+                # this check (since CancelledError is not a subclass of
+                # Exception then).
+                break
+            except Exception:
+                err_count += 1
+                # Only warn if multiple subsequent attempts fail, and only once
+                # per set of subsequent failed attempts. This way we're not
+                # excessively noisy during a connection blip, but we also don't
+                # silently fail.
+                if err_count == warn_at:
+                    logger.warning(
+                        "Failed to sync cluster info multiple times - perhaps "
+                        "there's a connection issue? Error:",
+                        exc_info=True,
+                    )
+            # Sleep, with error backoff
+            interval = min(max_interval, self._sync_interval * 1.5 ** err_count)
+            await asyncio.sleep(interval)
 
     async def _close(self):
         if self.status == Status.closed:
             return
 
+        self.status = Status.closing
+
         with suppress(AttributeError):
             self._adaptive.stop()
 
@@ -119,6 +152,11 @@ async def _close(self):
         if self._watch_worker_status_task:
             await self._watch_worker_status_task
 
+        if self._sync_cluster_info_task:
+            self._sync_cluster_info_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self._sync_cluster_info_task
+
         if self.scheduler_comm:
             await self.scheduler_comm.close_rpc()
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -7060,17 +7060,12 @@ def run_function(self, stream, function, args=(), kwargs={}, wait=True):
 
     def set_metadata(self, comm=None, keys=None, value=None):
         parent: SchedulerState = cast(SchedulerState, self)
-        try:
-            metadata = parent._task_metadata
-            for key in keys[:-1]:
-                if key not in metadata or not isinstance(metadata[key], (dict, list)):
-                    metadata[key] = {}
-                metadata = metadata[key]
-            metadata[keys[-1]] = value
-        except Exception:
-            import pdb
-
-            pdb.set_trace()
+        metadata = parent._task_metadata
+        for key in keys[:-1]:
+            if key not in metadata or not isinstance(metadata[key], (dict, list)):
+                metadata[key] = {}
+            metadata = metadata[key]
+        metadata[keys[-1]] = value
 
     def get_metadata(self, comm=None, keys=None, default=no_default):
         parent: SchedulerState = cast(SchedulerState, self)


Test patch:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1103,6 +1103,42 @@ async def test_cluster_info_sync():
         assert info["foo"] == "bar"
 
 
+@pytest.mark.asyncio
+async def test_cluster_info_sync_is_robust_to_network_blips(monkeypatch):
+    async with LocalCluster(
+        processes=False, asynchronous=True, scheduler_sync_interval="1ms"
+    ) as cluster:
+        assert cluster._cluster_info["name"] == cluster.name
+
+        error_called = False
+
+        async def error(*args, **kwargs):
+            nonlocal error_called
+            await asyncio.sleep(0.001)
+            error_called = True
+            raise OSError
+
+        # Temporarily patch the `set_metadata` RPC to error
+        with monkeypatch.context() as patch:
+            patch.setattr(cluster.scheduler_comm, "set_metadata", error)
+
+            # Set a new cluster_info value
+            cluster._cluster_info["foo"] = "bar"
+
+            # Wait for the bad method to be called at least once
+            while not error_called:
+                await asyncio.sleep(0.01)
+
+        # Check that cluster_info is resynced after the error condition is fixed
+        while "foo" not in cluster.scheduler.get_metadata(
+            keys=["cluster-manager-info"]
+        ):
+            await asyncio.sleep(0.01)
+
+        info = cluster.scheduler.get_metadata(keys=["cluster-manager-info"])
+        assert info["foo"] == "bar"
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("host", [None, "127.0.0.1"])
 @pytest.mark.parametrize("use_nanny", [True, False])