Repository: dask/distributed

Issue:
Sporadic hangs in distributed/tests/test_worker_failure.py::test_restart_sync
The new restart logic seems a bit fragile:
https://travis-ci.org/dask/distributed/jobs/242782855#L1608
```
distributed.core - ERROR - Worker not started

Traceback (most recent call last):
  File "/home/travis/build/dask/distributed/distributed/core.py", line 261, in handle_comm
    result = yield result
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/home/travis/build/dask/distributed/distributed/nanny.py", line 201, in restart
    yield self.instantiate()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/home/travis/build/dask/distributed/distributed/nanny.py", line 194, in instantiate
    yield self.process.start()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1063, in run
    yielded = self.gen.throw(*exc_info)
  File "/home/travis/build/dask/distributed/distributed/nanny.py", line 296, in start
    yield self._wait_until_running()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1055, in run
    value = future.result()
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 4, in raise_exc_info
  File "/home/travis/miniconda/envs/test-environment/lib/python3.5/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/travis/build/dask/distributed/distributed/nanny.py", line 381, in _wait_until_running
    raise ValueError("Worker not started")
ValueError: Worker not started
```



Discussion:

Resolving patch:
diff --git a/distributed/nanny.py b/distributed/nanny.py
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -217,10 +217,14 @@ def _on_exit(self, exitcode):
                     yield self._close()
                     return
 
-            if self.status not in ('closing', 'closed'):
-                if self.auto_restart:
-                    logger.warning("Restarting worker")
-                    yield self.instantiate()
+            try:
+                if self.status not in ('closing', 'closed'):
+                    if self.auto_restart:
+                        logger.warning("Restarting worker")
+                        yield self.instantiate()
+            except Exception:
+                logger.error("Failed to restart worker after its process exited",
+                             exc_info=True)
 
     @property
     def pid(self):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -835,7 +835,7 @@ def stimulus_missing_data(self, cause=None, key=None, worker=None,
 
             return {}
 
-    def remove_worker(self, comm=None, address=None, safe=False, restart=False):
+    def remove_worker(self, comm=None, address=None, safe=False, close=True):
         """
         Remove worker from cluster
 
@@ -855,58 +855,58 @@ def remove_worker(self, comm=None, address=None, safe=False, restart=False):
             self.log_event(['all', address], {'action': 'remove-worker',
                                               'worker': address})
             logger.info("Remove worker %s", address)
-            with ignoring(AttributeError, CommClosedError):
-                self.worker_comms[address].send({'op': 'close'})
-
-            if not restart:
-                self.host_info[host]['cores'] -= self.ncores[address]
-                self.host_info[host]['addresses'].remove(address)
-                self.total_ncores -= self.ncores[address]
-
-                if not self.host_info[host]['addresses']:
-                    del self.host_info[host]
-
-                del self.worker_comms[address]
-                del self.ncores[address]
-                self.workers.remove(address)
-                del self.aliases[self.worker_info[address]['name']]
-                del self.worker_info[address]
-                if address in self.idle:
-                    self.idle.remove(address)
-                if address in self.saturated:
-                    self.saturated.remove(address)
-
-                recommendations = OrderedDict()
-
-                in_flight = set(self.processing.pop(address))
-                for k in list(in_flight):
-                    # del self.rprocessing[k]
-                    if not safe:
-                        self.suspicious_tasks[k] += 1
-                    if not safe and self.suspicious_tasks[k] > self.allowed_failures:
-                        e = pickle.dumps(KilledWorker(k, address))
-                        r = self.transition(k, 'erred', exception=e, cause=k)
-                        recommendations.update(r)
-                        in_flight.remove(k)
-                    else:
-                        recommendations[k] = 'released'
+            if close:
+                with ignoring(AttributeError, CommClosedError):
+                    self.worker_comms[address].send({'op': 'close'})
+
+            self.host_info[host]['cores'] -= self.ncores[address]
+            self.host_info[host]['addresses'].remove(address)
+            self.total_ncores -= self.ncores[address]
+
+            if not self.host_info[host]['addresses']:
+                del self.host_info[host]
+
+            del self.worker_comms[address]
+            del self.ncores[address]
+            self.workers.remove(address)
+            del self.aliases[self.worker_info[address]['name']]
+            del self.worker_info[address]
+            if address in self.idle:
+                self.idle.remove(address)
+            if address in self.saturated:
+                self.saturated.remove(address)
 
-                self.total_occupancy -= self.occupancy.pop(address)
-                del self.worker_bytes[address]
-                self.remove_resources(address)
+            recommendations = OrderedDict()
 
-                for key in self.has_what.pop(address):
-                    self.who_has[key].remove(address)
-                    if not self.who_has[key]:
-                        if key in self.tasks:
-                            recommendations[key] = 'released'
-                        else:
-                            recommendations[key] = 'forgotten'
+            in_flight = set(self.processing.pop(address))
+            for k in list(in_flight):
+                # del self.rprocessing[k]
+                if not safe:
+                    self.suspicious_tasks[k] += 1
+                if not safe and self.suspicious_tasks[k] > self.allowed_failures:
+                    e = pickle.dumps(KilledWorker(k, address))
+                    r = self.transition(k, 'erred', exception=e, cause=k)
+                    recommendations.update(r)
+                    in_flight.remove(k)
+                else:
+                    recommendations[k] = 'released'
 
-                self.transitions(recommendations)
+            self.total_occupancy -= self.occupancy.pop(address)
+            del self.worker_bytes[address]
+            self.remove_resources(address)
+
+            for key in self.has_what.pop(address):
+                self.who_has[key].remove(address)
+                if not self.who_has[key]:
+                    if key in self.tasks:
+                        recommendations[key] = 'released'
+                    else:
+                        recommendations[key] = 'forgotten'
+
+            self.transitions(recommendations)
 
-                if self.validate:
-                    assert all(self.who_has.values()), len(self.who_has)
+            if self.validate:
+                assert all(self.who_has.values()), len(self.who_has)
 
             for plugin in self.plugins[:]:
                 try:
@@ -1487,11 +1487,13 @@ def restart(self, client=None, timeout=3):
                 self.client_releases_keys(keys=keys, client=client)
 
             nannies = {addr: self.get_worker_service_addr(addr, 'nanny')
-                       for addr in self.worker_info}
+                       for addr in self.workers}
 
             for addr in list(self.workers):
                 try:
-                    self.remove_worker(address=addr, restart=True)
+                    # Ask the worker to close if it doesn't have a nanny,
+                    # otherwise the nanny will kill it anyway
+                    self.remove_worker(address=addr, close=addr not in nannies)
                 except Exception as e:
                     logger.info("Exception while restarting.  This is normal",
                                 exc_info=True)
@@ -1502,15 +1504,14 @@ def restart(self, client=None, timeout=3):
             for collection in self._worker_collections:
                 collection.clear()
 
-
-            logger.debug("Send kill signal to nannies: %s", nannies)
-
             for plugin in self.plugins[:]:
                 try:
                     plugin.restart(self)
                 except Exception as e:
                     logger.exception(e)
 
+            logger.debug("Send kill signal to nannies: %s", nannies)
+
             nannies = [rpc(nanny_address)
                        for nanny_address in nannies.values()
                        if nanny_address is not None]
diff --git a/distributed/worker.py b/distributed/worker.py
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -334,8 +334,8 @@ def _close(self, report=True, timeout=10, nanny=True):
 
         self.status = 'closed'
 
-        if nanny and 'nanny' in self.services:
-            with self.rpc(self.services['nanny']) as r:
+        if nanny and 'nanny' in self.service_ports:
+            with self.rpc((self.ip, self.service_ports['nanny'])) as r:
                 yield r.terminate()
 
         self.rpc.close()


Test patch: