Repository: dask/distributed

Issue:
Client reconnect should have a timeout
Currently if a scheduler goes down, but the client isn't closed, the client will keep trying to connect to the scheduler address forever. This should probably have a timeout - maybe the same timeout as the initial connect (`distributed.timeouts.connect`)?

Relevant code:

https://github.com/dask/distributed/blob/b36e87d14e75c0cbdd0e3292ac65db2a836f74bd/distributed/client.py#L871-L876


Discussion:
The client keeps around the timeout provided at initialization at `self._timeout`.  This defaults to `defaults.timeouts.connect`.  Would that work?
Yeah, that'd be good. I'll try to get a PR up for this sometime soon.

Resolving patch:
diff --git a/distributed/client.py b/distributed/client.py
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -954,9 +954,10 @@ def _start(self, timeout=no_default, **kwargs):
         raise gen.Return(self)
 
     @gen.coroutine
-    def _reconnect(self, timeout=0.1):
+    def _reconnect(self):
         with log_errors():
             assert self.scheduler_comm.comm.closed()
+
             self.status = "connecting"
             self.scheduler_comm = None
 
@@ -964,12 +965,23 @@ def _reconnect(self, timeout=0.1):
                 st.cancel()
             self.futures.clear()
 
-            while self.status == "connecting":
+            timeout = self._timeout
+            deadline = self.loop.time() + timeout
+            while timeout > 0 and self.status == "connecting":
                 try:
-                    yield self._ensure_connected()
+                    yield self._ensure_connected(timeout=timeout)
                     break
                 except EnvironmentError:
-                    yield gen.sleep(timeout)
+                    # Wait a bit before retrying
+                    yield gen.sleep(0.1)
+                    timeout = deadline - self.loop.time()
+            else:
+                logger.error(
+                    "Failed to reconnect to scheduler after %.2f "
+                    "seconds, closing client",
+                    self._timeout,
+                )
+                yield self._close()
 
     @gen.coroutine
     def _ensure_connected(self, timeout=None):


Test patch:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3704,6 +3704,19 @@ def test_reconnect(loop):
     c.close()
 
 
+@gen_cluster(client=True, ncores=[], client_kwargs={"timeout": 0.5})
+def test_reconnect_timeout(c, s):
+    with captured_logger(logging.getLogger("distributed.client")) as logger:
+        yield s.close()
+        start = time()
+        while c.status != "closed":
+            yield c._update_scheduler_info()
+            yield gen.sleep(0.05)
+            assert time() < start + 5, "Timeout waiting for reconnect to fail"
+    text = logger.getvalue()
+    assert "Failed to reconnect" in text
+
+
 @slow
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="num_fds not supported on windows"
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -974,7 +974,7 @@ def coro():
                                 if s.validate:
                                     s.validate_state()
                             finally:
-                                if client:
+                                if client and c.status not in ("closing", "closed"):
                                     yield c._close(fast=s.status == "closed")
                                 yield end_cluster(s, workers)
                                 yield gen.with_timeout(