Repository: dask/distributed

Issue:
Track killed workers in SpecCluster
I've noticed that workers created by a `SpecCluster` remain in the worker count if they are killed harshly. The client shows them as being gone.

Perhaps we should add some checks in correct state to remove any dead workers and call it intermittently?


Discussion:
The relevant code to fix this is probably here:

https://github.com/dask/distributed/blob/b083b10d64763b38e559096127d6e3e0c0638c31/distributed/scheduler.py#L5022-L5053

and here

https://github.com/dask/distributed/blob/b083b10d64763b38e559096127d6e3e0c0638c31/distributed/deploy/cluster.py#L90-L108

Probably we want to override `Cluster. _watch_worker_status` with `SpecCluster. _watch_worker_status` that also removes the object from `.workers`?
OK, I'm working on this, will submit a PR soon.

Resolving patch:
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -8,6 +8,7 @@
 
 import click
 import dask
+from dask.utils import ignoring
 from distributed import Nanny, Worker
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
@@ -354,6 +355,9 @@ def del_pid_file():
             "dask-worker SCHEDULER_ADDRESS:8786"
         )
 
+    with ignoring(TypeError, ValueError):
+        name = int(name)
+
     nannies = [
         t(
             scheduler,
@@ -367,7 +371,7 @@ def del_pid_file():
             port=port,
             dashboard_address=dashboard_address if dashboard else None,
             service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
-            name=name if nprocs == 1 or not name else name + "-" + str(i),
+            name=name if nprocs == 1 or not name else str(name) + "-" + str(i),
             **kwargs
         )
         for i in range(nprocs)
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -95,18 +95,22 @@ async def _watch_worker_status(self, comm):
             except OSError:
                 break
 
-            for op, msg in msgs:
-                if op == "add":
-                    workers = msg.pop("workers")
-                    self.scheduler_info["workers"].update(workers)
-                    self.scheduler_info.update(msg)
-                elif op == "remove":
-                    del self.scheduler_info["workers"][msg]
-                else:
-                    raise ValueError("Invalid op", op, msg)
+            with log_errors():
+                for op, msg in msgs:
+                    self._update_worker_status(op, msg)
 
         await comm.close()
 
+    def _update_worker_status(self, op, msg):
+        if op == "add":
+            workers = msg.pop("workers")
+            self.scheduler_info["workers"].update(workers)
+            self.scheduler_info.update(msg)
+        elif op == "remove":
+            del self.scheduler_info["workers"][msg]
+        else:
+            raise ValueError("Invalid op", op, msg)
+
     def adapt(self, Adaptive=Adaptive, **kwargs) -> Adaptive:
         """ Turn on adaptivity
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,18 +1,23 @@
 import asyncio
 import atexit
 import copy
+import logging
 import math
 import weakref
 
+import dask
 from tornado import gen
 
 from .cluster import Cluster
 from ..core import rpc, CommClosedError
-from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes
+from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes, parse_timedelta
 from ..scheduler import Scheduler
 from ..security import Security
 
 
+logger = logging.getLogger(__name__)
+
+
 class ProcessInterface:
     """
     An interface for Scheduler and Worker processes for use in SpecCluster
@@ -201,6 +206,7 @@ def __init__(
         self._i = 0
         self.security = security or Security()
         self.scheduler_comm = None
+        self._futures = set()
 
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
@@ -267,13 +273,14 @@ async def _correct_state_internal(self):
             if to_close:
                 if self.scheduler.status == "running":
                     await self.scheduler_comm.retire_workers(workers=list(to_close))
-                tasks = [self.workers[w].close() for w in to_close]
+                tasks = [self.workers[w].close() for w in to_close if w in self.workers]
                 await asyncio.wait(tasks)
                 for task in tasks:  # for tornado gen.coroutine support
                     with ignoring(RuntimeError):
                         await task
             for name in to_close:
-                del self.workers[name]
+                if name in self.workers:
+                    del self.workers[name]
 
             to_open = set(self.worker_spec) - set(self.workers)
             workers = []
@@ -293,6 +300,22 @@ async def _correct_state_internal(self):
                     await w  # for tornado gen.coroutine support
             self.workers.update(dict(zip(to_open, workers)))
 
+    def _update_worker_status(self, op, msg):
+        if op == "remove":
+            name = self.scheduler_info["workers"][msg]["name"]
+
+            def f():
+                if name in self.workers and msg not in self.scheduler_info:
+                    self._futures.add(asyncio.ensure_future(self.workers[name].close()))
+                    del self.workers[name]
+
+            delay = parse_timedelta(
+                dask.config.get("distributed.deploy.lost-worker-timeout")
+            )
+
+            asyncio.get_event_loop().call_later(delay, f)
+        super()._update_worker_status(op, msg)
+
     def __await__(self):
         async def _():
             if self.status == "created":
@@ -314,13 +337,15 @@ async def _close(self):
 
         self.scale(0)
         await self._correct_state()
+        for future in self._futures:
+            await future
         async with self._lock:
             with ignoring(CommClosedError):
                 await self.scheduler_comm.close(close_workers=True)
 
         await self.scheduler.close()
         for w in self._created:
-            assert w.status == "closed"
+            assert w.status == "closed", w.status
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)


Test patch:
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -1,3 +1,4 @@
+import asyncio
 import pytest
 from click.testing import CliRunner
 
@@ -9,11 +10,11 @@
 from time import sleep
 
 import distributed.cli.dask_worker
-from distributed import Client
+from distributed import Client, Scheduler
 from distributed.metrics import time
 from distributed.utils import sync, tmpfile
 from distributed.utils_test import popen, terminate_process, wait_for_port
-from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import loop, cleanup  # noqa: F401
 
 
 def test_nanny_worker_ports(loop):
@@ -330,3 +331,13 @@ def test_bokeh_deprecation():
         except ValueError:
             # didn't pass scheduler
             pass
+
+
+@pytest.mark.asyncio
+async def test_integer_names(cleanup):
+    async with Scheduler(port=0) as s:
+        with popen(["dask-worker", s.address, "--name", "123"]) as worker:
+            while not s.workers:
+                await asyncio.sleep(0.01)
+            [ws] = s.workers.values()
+            assert ws.name == 123
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,8 +1,9 @@
 import asyncio
-from time import time
 
+import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters, ProcessInterface
+from distributed.metrics import time
 from distributed.utils_test import loop, cleanup  # noqa: F401
 from distributed.utils import is_valid_xml
 import toolz
@@ -124,6 +125,37 @@ async def test_scale(cleanup):
         assert len(cluster.workers) == 1
 
 
+@pytest.mark.asyncio
+async def test_unexpected_closed_worker(cleanup):
+    worker = {"cls": Worker, "options": {"nthreads": 1}}
+    with dask.config.set({"distributed.deploy.lost-worker-timeout": "10ms"}):
+        async with SpecCluster(
+            asynchronous=True, scheduler=scheduler, worker=worker
+        ) as cluster:
+            assert not cluster.workers
+            assert not cluster.worker_spec
+
+            # Scale up
+            cluster.scale(2)
+            assert not cluster.workers
+            assert cluster.worker_spec
+
+            await cluster
+            assert len(cluster.workers) == 2
+
+            # Close one
+            await list(cluster.workers.values())[0].close()
+            start = time()
+            while len(cluster.workers) > 1:  # wait for messages to flow around
+                await asyncio.sleep(0.01)
+                assert time() < start + 2
+            assert len(cluster.workers) == 1
+            assert len(cluster.worker_spec) == 2
+
+            await cluster
+            assert len(cluster.workers) == 2
+
+
 @pytest.mark.asyncio
 async def test_broken_worker():
     with pytest.raises(Exception) as info: