Repository: dask/distributed

Issue:
Distributed.joblib code fails to affect sklearn
The comments in [this stackoverflow answer](http://stackoverflow.com/questions/38601026/easy-way-to-use-parallel-options-of-scikit-learn-functions-on-hpc/38814491#38814491) raise concerns about the effectiveness of `distributed.joblib` to parallelize vanilla sklearn code.  It appears that sklearn ships with its own version of Joblib, which the plugin registration in `distributed.joblib` does not affect.

It would be good to test sklearn functionality and, if necessary, add plugin registration to `sklearn.externals.joblib` in the same way we do to normal `joblib`.



Discussion:

Resolving patch:
diff --git a/distributed/joblib.py b/distributed/joblib.py
--- a/distributed/joblib.py
+++ b/distributed/joblib.py
@@ -1,10 +1,36 @@
 from __future__ import print_function, division, absolute_import
 
-from joblib._parallel_backends import ParallelBackendBase, AutoBatchingMixin
-from joblib.parallel import register_parallel_backend
+from distutils.version import LooseVersion
+
 from tornado import gen
 
 from .executor import Executor, _wait
+from .utils import ignoring
+
+
+# A user could have installed joblib, sklearn, both, or neither. Further, only
+# joblib >= 0.10.0 supports backends, so we also need to check for that. This
+# bit of logic is to ensure that we create and register the backend for all
+# viable installations of joblib.
+joblib = sk_joblib = None
+with ignoring(ImportError):
+    import joblib
+    if LooseVersion(joblib.__version__) < '0.10.0':
+        joblib = None
+with ignoring(ImportError):
+    import sklearn.externals.joblib as sk_joblib
+    if LooseVersion(sk_joblib.__version__) < '0.10.0':
+        sk_joblib = None
+
+if joblib:
+    from joblib._parallel_backends import (ParallelBackendBase,
+        AutoBatchingMixin)
+elif sk_joblib:
+    from sklearn.externals.joblib._parallel_backends import (
+        ParallelBackendBase, AutoBatchingMixin)
+else:
+    raise RuntimeError("Joblib backend requires either `joblib` >= '0.10.0' "
+                       " or `sklearn` > '0.17.1'. Please install or upgrade")
 
 
 class DistributedBackend(ParallelBackendBase, AutoBatchingMixin):
@@ -45,4 +71,8 @@ def abort_everything(self, ensure_ready=True):
         self.futures.clear()
 
 
-register_parallel_backend('distributed', DistributedBackend)
+# Register the backend with any available versions of joblib
+if joblib:
+    joblib.register_parallel_backend('distributed', DistributedBackend)
+if sk_joblib:
+    sk_joblib.register_parallel_backend('distributed', DistributedBackend)


Test patch:
diff --git a/distributed/tests/test_joblib.py b/distributed/tests/test_joblib.py
--- a/distributed/tests/test_joblib.py
+++ b/distributed/tests/test_joblib.py
@@ -1,14 +1,13 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
-pytest.importorskip('joblib')
 from random import random
+from time import sleep
 
-from joblib import parallel_backend, Parallel, delayed
-
-import distributed.joblib
 from distributed.utils_test import inc, cluster, loop
-from time import sleep
+
+backend = pytest.importorskip('distributed.joblib')
+joblibs = [backend.joblib, backend.sk_joblib]
 
 
 def slow_raise_value_error(condition, duration=0.05):
@@ -17,37 +16,46 @@ def slow_raise_value_error(condition, duration=0.05):
         raise ValueError("condition evaluated to True")
 
 
-def test_simple(loop):
+@pytest.mark.parametrize('joblib', joblibs)
+def test_simple(loop, joblib):
+    if joblib is None:
+        pytest.skip()
+    Parallel = joblib.Parallel
+    delayed = joblib.delayed
     with cluster() as (s, [a, b]):
-        with parallel_backend('distributed', loop=loop,
-                scheduler_host=('127.0.0.1', s['port'])) as p:
+        with joblib.parallel_backend('distributed', loop=loop,
+                scheduler_host=('127.0.0.1', s['port'])):
 
             seq = Parallel()(delayed(inc)(i) for i in range(10))
             assert seq == [inc(i) for i in range(10)]
 
             with pytest.raises(ValueError):
                 Parallel()(delayed(slow_raise_value_error)(i == 3)
-                    for i in range(10))
+                           for i in range(10))
 
             seq = Parallel()(delayed(inc)(i) for i in range(10))
             assert seq == [inc(i) for i in range(10)]
 
-            from joblib.parallel import get_active_backend
-            ba, _ = get_active_backend()
+            ba, _ = joblib.parallel.get_active_backend()
             ba.executor.shutdown()
 
 
 def random2():
     return random()
 
-def test_dont_assume_function_purity(loop):
+
+@pytest.mark.parametrize('joblib', joblibs)
+def test_dont_assume_function_purity(loop, joblib):
+    if joblib is None:
+        pytest.skip()
+    Parallel = joblib.Parallel
+    delayed = joblib.delayed
     with cluster() as (s, [a, b]):
-        with parallel_backend('distributed', loop=loop,
-                scheduler_host=('127.0.0.1', s['port'])) as p:
+        with joblib.parallel_backend('distributed', loop=loop,
+                scheduler_host=('127.0.0.1', s['port'])):
 
             x, y = Parallel()(delayed(random2)() for i in range(2))
             assert x != y
 
-            from joblib.parallel import get_active_backend
-            ba, _ = get_active_backend()
+            ba, _ = joblib.parallel.get_active_backend()
             ba.executor.shutdown()