Repository: dask/distributed

Issue:
self.workers.remove(w) fails trying to remove worker.
Probably a nit, but this code:

```python
import dask.array as da

from distributed import (Client,
    LocalCluster)

import numpy as np

cluster = LocalCluster()
client = Client(cluster.scheduler_address)

ones = da.ones(shape=(1000,1000), chunks=(100,100), dtype=np.float64)

A = client.persist(ones)

print A

f = client.compute(A)

print f.result()

client.shutdown()
cluster.close()
```

fails in the following way:

```bash
dask.array<wrapped, shape=(1000, 1000), dtype=float64, chunksize=(100, 100)>
[[ 1.  1.  1. ...,  1.  1.  1.]
 [ 1.  1.  1. ...,  1.  1.  1.]
 [ 1.  1.  1. ...,  1.  1.  1.]
 ..., 
 [ 1.  1.  1. ...,  1.  1.  1.]
 [ 1.  1.  1. ...,  1.  1.  1.]
 [ 1.  1.  1. ...,  1.  1.  1.]]
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0b0f24c8>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0a012af8>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0a012db8>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0b12d158>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0a021e68>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b09f70368>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0b0f28e8>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
tornado.application - ERROR - Exception in callback <functools.partial object at 0x7f7b0b0f2d08>
Traceback (most recent call last):
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 605, in _run_callback
    ret = callback()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/ioloop.py", line 626, in _discard_future_result
    future.result()
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/concurrent.py", line 238, in result
    raise_exc_info(self._exc_info)
  File "/home/sperkins/venv/mb/local/lib/python2.7/site-packages/tornado/gen.py", line 1069, in run
    yielded = self.gen.send(value)
  File "/home/sperkins/work/ska/code/distributed/distributed/deploy/local.py", line 206, in _stop_worker
    self.workers.remove(w)
ValueError: list.remove(x): x not in list
```


Discussion:
Could you guard your code with a `if __name__ == "__main__"` statement and try again?
> Could you guard your code with a `if __name__ == "__main__"` statement and try again?

It still occurs


Resolving patch:
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -203,7 +203,8 @@ def start_worker(self, ncores=0, **kwargs):
     @gen.coroutine
     def _stop_worker(self, w):
         yield w._close()
-        self.workers.remove(w)
+        if w in self.workers:
+            self.workers.remove(w)
 
     def stop_worker(self, w):
         """ Stop a running worker


Test patch:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -15,7 +15,8 @@
 from distributed.deploy.local import LocalCluster
 from distributed.metrics import time
 from distributed.utils_test import (inc, loop, raises, gen_test, pristine_loop,
-        assert_can_connect_locally_4, assert_can_connect_from_everywhere_4_6)
+        assert_can_connect_locally_4, assert_can_connect_from_everywhere_4_6,
+        captured_logger)
 from distributed.utils import ignoring, sync
 from distributed.worker import TOTAL_MEMORY, _ncores
 
@@ -32,6 +33,20 @@ def test_simple(loop):
             assert any(w.data == {x.key: 2} for w in c.workers)
 
 
+def test_close_twice(loop):
+    cluster = LocalCluster()
+    with Client(cluster.scheduler_address) as client:
+        f = client.map(inc, range(100))
+        client.gather(f)
+    with captured_logger('tornado.application') as log:
+        cluster.close()
+        cluster.close()
+        sleep(0.5)
+    log = log.getvalue()
+    print(log)
+    assert not log
+
+
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='multi-loop')
 def test_procs(loop):
     with LocalCluster(2, scheduler_port=0, processes=False, threads_per_worker=3,
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -797,6 +797,8 @@ def assert_can_connect_locally_6(port, timeout=None, connection_args=None):
 def captured_logger(logger):
     """Capture output from the given Logger.
     """
+    if isinstance(logger, str):
+        logger = logging.getLogger(logger)
     orig_handlers = logger.handlers[:]
     sio = six.StringIO()
     logger.handlers[:] = [logging.StreamHandler(sio)]