Repository: dask/distributed

Issue:
distributed.get_client() can return a closed client
In the following reproducer:
```python
from distributed import get_client, Client, LocalCluster
import scipy  # required to trigger the bug, probably by slowing the gc


def inc(i):
    return i + 1


if __name__ == "__main__":
    cluster = LocalCluster(n_workers=0, processes=False, threads_per_worker=2)

    c = Client(cluster)
    cluster.scale(2)
    assert c == get_client()
    print(cluster)

    c.close()
    cluster.close()
    # import gc; gc.collect()

    cluster2 = LocalCluster(n_workers=0, processes=False, threads_per_worker=2)
    c2 = Client(cluster2)
    cluster2.scale(2)
    print(cluster2)

    current_client = get_client()
    assert c2 == current_client

    print("OK")
```

I create one Cluster and bind a client to it. After closing them, I re-create a new cluster and a new client, but `get_client()` tries to connect to the old, closed client -- because it has been closed, an `OSError` is raised:
```python-traceback
Traceback (most recent call last):                                                                                                                                                                                 
  File "/home/pierreglaser/repos/distributed/distributed/comm/core.py", line 218, in connect                                                                                                                       
    quiet_exceptions=EnvironmentError,                                                                                                                                                                             
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1133, in run                                                                                                 
    value = future.result()                                                                                                                                                                                        
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 326, in wrapper                                                                                              
    yielded = next(result)                                                                                                                                                                                         
  File "/home/pierreglaser/repos/distributed/distributed/comm/inproc.py", line 296, in connect                                                                                                                     
    raise IOError("no endpoint for inproc address %r" % (address,))                                                                                                                                                
OSError: no endpoint for inproc address '10.200.21.15/24632/1'                                                                                                                                                     
                                                                                                                                                                                                                   
During handling of the above exception, another exception occurred:                                                                                                                                                
                                                                                                                                                                                                                   
Traceback (most recent call last):                                                                                                                                                                                 
  File "test_client_distributed.py", line 26, in <module>                                                                                                                                                          
    current_client = get_client()                                                                                                                                                                                  
  File "/home/pierreglaser/repos/distributed/distributed/worker.py", line 2911, in get_client                                                                                                                      
    return worker._get_client(timeout=timeout)                                                                                                                                                                     
  File "/home/pierreglaser/repos/distributed/distributed/worker.py", line 2810, in _get_client                                                                                                                     
    timeout=timeout,                                                                                                                                                                                               
  File "/home/pierreglaser/repos/distributed/distributed/client.py", line 722, in __init__                                                                                                                         
    self.start(timeout=timeout)                                                                                                                                                                                    
  File "/home/pierreglaser/repos/distributed/distributed/client.py", line 872, in start                                                                                                                            
    sync(self.loop, self._start, **kwargs)                                                                                                                                                                         
  File "/home/pierreglaser/repos/distributed/distributed/utils.py", line 332, in sync                                                                                                                              
    six.reraise(*error[0])                                                                                                                                                                                         
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/six.py", line 693, in reraise                                                                                                      
    raise value                                                                                                                                                                                                    
  File "/home/pierreglaser/repos/distributed/distributed/utils.py", line 317, in f                                                                                                                                 
    result[0] = yield future                                                                                                                                                                                       
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1133, in run                                                                                                 
    value = future.result()                                                                                                                                                                                        
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1141, in run                                                                                                 
    yielded = self.gen.throw(*exc_info)                                                                                                                                                                            
  File "/home/pierreglaser/repos/distributed/distributed/client.py", line 968, in _start                                                                                                                           
    yield self._ensure_connected(timeout=timeout)                                                                                                                                                                  
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1133, in run                                                                                                 
    value = future.result()                                                                                                                                                                                        
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1141, in run                                                                                                 
    yielded = self.gen.throw(*exc_info)                                                                                                                                                                            
  File "/home/pierreglaser/repos/distributed/distributed/client.py", line 1024, in _ensure_connected                                                                                                               
    connection_args=self.connection_args,                                                                                                                                                                          
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1133, in run                                                                                                 
    value = future.result()                                                                                                                                                                                        
  File "/home/pierreglaser/.virtualenvs/joblib_py37/lib/python3.7/site-packages/tornado/gen.py", line 1141, in run                                                                                                 
    yielded = self.gen.throw(*exc_info)                                                                                                                                                                            
  File "/home/pierreglaser/repos/distributed/distributed/comm/core.py", line 228, in connect                                                                                                                       
    _raise(error)                                                                                                                                                                                                  
  File "/home/pierreglaser/repos/distributed/distributed/comm/core.py", line 207, in _raise                                                                                                                        
    raise IOError(msg)                                                                                                                      
```


Discussion:

Resolving patch:
diff --git a/distributed/worker.py b/distributed/worker.py
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2857,7 +2857,7 @@ def get_worker():
         return thread_state.execution_state["worker"]
     except AttributeError:
         try:
-            return first(Worker._instances)
+            return first(w for w in Worker._instances if w.status == "running")
         except StopIteration:
             raise ValueError("No workers found")
 


Test patch:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -14,7 +14,7 @@
 from tornado import gen
 import pytest
 
-from distributed import Client, Worker, Nanny
+from distributed import Client, Worker, Nanny, get_client
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils_test import (
@@ -831,3 +831,26 @@ def test_starts_up_sync(loop):
         assert len(cluster.scheduler.workers) == 2
     finally:
         cluster.close()
+
+
+def test_dont_select_closed_worker():
+    # Make sure distributed does not try to reuse a client from a
+    # closed cluster (https://github.com/dask/distributed/issues/2840).
+    with clean(threads=False):
+        cluster = LocalCluster(n_workers=0)
+        c = Client(cluster)
+        cluster.scale(2)
+        assert c == get_client()
+
+        c.close()
+        cluster.close()
+
+        cluster2 = LocalCluster(n_workers=0)
+        c2 = Client(cluster2)
+        cluster2.scale(2)
+
+        current_client = get_client()
+        assert c2 == current_client
+
+        cluster2.close()
+        c2.close()