Repository: dask/distributed

Issue:
distributed.joblib fails with LocalCluster and processes=False
```python
import time
import joblib
import distributed.joblib
from dask.distributed import Client

client = Client(processes=False)
client

def f(x):
    time.sleep(0.5)
    return x

with joblib.parallel_backend('dask'):
    joblib.Parallel(n_jobs=4)(joblib.delayed(f)(i) for i in range(10))
````

Fails with

```pytb
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-1-73a587b0ff66> in <module>()
     12 
     13 with joblib.parallel_backend('dask'):
---> 14     joblib.Parallel(n_jobs=4)(joblib.delayed(f)(i) for i in range(10))

~/sandbox/joblib/joblib/parallel.py in __call__(self, iterable)
    881                 self._iterating = False
    882 
--> 883             with self._backend.retrieval_context():
    884                 self.retrieve()
    885             # Make sure that we get a last message telling us we are done

/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py in __enter__(self)
     80     def __enter__(self):
     81         try:
---> 82             return next(self.gen)
     83         except StopIteration:
     84             raise RuntimeError("generator didn't yield") from None

~/Envs/dask-dev/lib/python3.6/site-packages/distributed/distributed/joblib.py in retrieval_context(self)
    181         # this is used.
    182         try:
--> 183             secede()
    184         except ValueError:
    185             # We are not a worker, i.e. not in nested parallelism.

~/Envs/dask-dev/lib/python3.6/site-packages/distributed/distributed/worker.py in secede()
   2641     """
   2642     worker = get_worker()
-> 2643     tpe_secede()  # have this thread secede from the thread pool
   2644     duration = time() - thread_state.start_time
   2645     worker.loop.add_callback(worker.maybe_transition_long_running,

~/Envs/dask-dev/lib/python3.6/site-packages/distributed/distributed/threadpoolexecutor.py in secede(adjust)
    102     thread_state.proceed = False
    103     with threads_lock:
--> 104         thread_state.executor._threads.remove(threading.current_thread())
    105         if adjust:
    106             thread_state.executor._adjust_thread_count()

AttributeError: '_thread._local' object has no attribute 'executor'
```

Will look more closely later.


Discussion:
I am not yet able to reproduce.

My first guess is that that is being called in something other than a task.
In case it helps:  this is joblib master (14784b711f2e777e87c857c5a0e8fbd9d5849936) and distributed master

```
In [4]: distributed.__version__
Out[4]: '1.21.0+10.g7b21143c'
```

> My first guess is that that is being called in something other than a task.

That seems likely. Does catching `AttributeError` seem like the cleanest way to write a function that can be run on a worker or standalone?

```python
def f(x):
    try:
        secede()
    except AttributeError:
        pass

    time.sleep(1)

    try:
        rejoin()
    except AttributeError:
        pass
    return x
```

That seems a bit verbose. I wonder if we could provide a version of secede or rejoin (perhaps a context manager) that makes things a bit nicer? Or is there a better way to test for certain whether you're a dask worker, and only secede / rejoin if that's the case?
OK, reproduced.  

If you wanted to be slightly more precise you could also check for the following:

```python
from .worker import thread_state
if hasattr(thread_state, 'execution_state'):
    ...
```
This would also be a reasonable function to make top-level along with `get_worker` and `get_client`

Resolving patch:
diff --git a/distributed/joblib.py b/distributed/joblib.py
--- a/distributed/joblib.py
+++ b/distributed/joblib.py
@@ -10,6 +10,7 @@
 from .client import Client, _wait
 from .utils import ignoring, funcname, itemgetter
 from . import get_client, secede, rejoin
+from .worker import thread_state
 
 # A user could have installed joblib, sklearn, both, or neither. Further, only
 # joblib >= 0.10.0 supports backends, so we also need to check for that. This
@@ -179,18 +180,14 @@ def retrieval_context(self):
         """
         # See 'joblib.Parallel.__call__' and 'joblib.Parallel.retrieve' for how
         # this is used.
-        try:
+        if hasattr(thread_state, 'execution_state'):
+            # we are in a worker. Secede to avoid deadlock.
             secede()
-        except ValueError:
-            # We are not a worker, i.e. not in nested parallelism.
-            pass
 
         yield
 
-        try:
+        if hasattr(thread_state, 'execution_state'):
             rejoin()
-        except AttributeError:
-            pass
 
 
 for base in _bases:


Test patch:
diff --git a/distributed/tests/test_joblib.py b/distributed/tests/test_joblib.py
--- a/distributed/tests/test_joblib.py
+++ b/distributed/tests/test_joblib.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 import os
+import importlib
 from distutils.version import LooseVersion
 
 import pytest
@@ -11,20 +12,32 @@
 from distributed.utils_test import loop # noqa F401
 
 distributed_joblib = pytest.importorskip('distributed.joblib')
-joblibs = [distributed_joblib.joblib, distributed_joblib.sk_joblib]
 joblib_funcname = distributed_joblib.joblib_funcname
 
 
+@pytest.fixture(params=['joblib', 'sk_joblib'])
+def joblib(request):
+    if request.param == 'joblib':
+        try:
+            this_joblib = importlib.import_module('joblib')
+        except ImportError:
+            pytest.skip("joblib not available")
+    else:
+        try:
+            this_joblib = importlib.import_module("sklearn.externals.joblib")
+        except ImportError:
+            pytest.skip("sklearn.externals.joblib not available")
+
+    return this_joblib
+
+
 def slow_raise_value_error(condition, duration=0.05):
     sleep(duration)
     if condition:
         raise ValueError("condition evaluated to True")
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_simple(loop, joblib):
-    if joblib is None:
-        pytest.skip()
     Parallel = joblib.Parallel
     delayed = joblib.delayed
     with cluster() as (s, [a, b]):
@@ -45,10 +58,7 @@ def random2():
     return random()
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_dont_assume_function_purity(loop, joblib):
-    if joblib is None:
-        pytest.skip()
     Parallel = joblib.Parallel
     delayed = joblib.delayed
     with cluster() as (s, [a, b]):
@@ -58,10 +68,7 @@ def test_dont_assume_function_purity(loop, joblib):
                 assert x != y
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_joblib_funcname(joblib):
-    if joblib is None:
-        pytest.skip()
     BatchedCalls = joblib.parallel.BatchedCalls
     if LooseVersion(joblib.__version__) <= "0.11.0":
         func = BatchedCalls([(random2,), (random2,)])
@@ -71,11 +78,7 @@ def test_joblib_funcname(joblib):
     assert joblib_funcname(random2) == 'random2'
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_joblib_backend_subclass(joblib):
-    if joblib is None:
-        pytest.skip()
-
     assert issubclass(distributed_joblib.DaskDistributedBackend,
                       joblib.parallel.ParallelBackendBase)
 
@@ -99,11 +102,7 @@ def __reduce__(self):
         return (CountSerialized, (self.x,))
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_joblib_scatter(loop, joblib):
-    if joblib is None:
-        pytest.skip()
-
     Parallel = joblib.Parallel
     delayed = joblib.delayed
 
@@ -137,10 +136,9 @@ def test_joblib_scatter(loop, joblib):
     assert z.count == 4
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_nested_backend_context_manager(loop, joblib):
-    if joblib is None or LooseVersion(joblib.__version__) <= "0.11.0":
-        pytest.skip("Joblib >= 0.11.1 required.")
+    if LooseVersion(joblib.__version__) <= "0.11.0":
+        pytest.skip("Joblib >= 0.11.1 required for nested parallelism.")
     Parallel = joblib.Parallel
     delayed = joblib.delayed
 
@@ -170,13 +168,20 @@ def get_nested_pids():
                     assert len(set(pid_group)) <= 2
 
 
-@pytest.mark.parametrize('joblib', joblibs)
 def test_errors(loop, joblib):
-    if joblib is None:
-        pytest.skip()
-
     with pytest.raises(ValueError) as info:
         with joblib.parallel_backend('dask'):
             pass
 
     assert "create a dask client" in str(info.value).lower()
+
+
+def test_secede_with_no_processes(loop, joblib):
+    # https://github.com/dask/distributed/issues/1775
+
+    def f(x):
+        return x
+
+    with Client(loop=loop, processes=False, set_as_default=True):
+        with joblib.parallel_backend('dask'):
+            joblib.Parallel(n_jobs=4)(joblib.delayed(f)(i) for i in range(2))