Repository: dask/dask

Issue:
BUG: multiprocessing scheduler silently hangs with lock task argument 
When passing a multiprocessing lock and using the multiprocessing scheduler in a dask graph (without even using the lock) dask silently hangs when `multiprocessing.get` is called.

PR #1341 adds a xfail test that proves this point



Discussion:
Multiprocessing locks aren't serializable, but `cloudpickle.dumps(lock)`, won't fail (although it probably should). It _will_ fail on load though, which tears down the worker process, causing the scheduler to hang.

What is your expected behavior here? A lock exists only in a process - serializing it between processes (if that worked) wouldn't have inter-process locking. A few options:
- Use `multiprocessing.Manager.Lock`. These _are_ serializable between processes, and will result in inter-process locking. See https://github.com/dask/dask/issues/1066#issuecomment-203106004 for an example.
- Use [locket](https://github.com/mwilliamson/locket.py) for file based locks (this is what partd uses).

If you only need locking in each process, but don't care if many processes access the same resource concurrently, then a small wrapper class around `multiprocessing.Lock` that creates a new lock when unpickled would probably work.

Thanks of the in-depth explanation!

This became an issue in #1293, and the desired behavior there was to lock multiple processes from using the same resource (an hdf file) simultaneously.
@mrocklin already suggested locket, but unfortunately locket uses `threading.Lock` which isn't serializable and fails when used with python's multiprocessing module.

AFAIK using multiprocessing.Lock is supposed to be share-able between multiple processes by using a single global lock instead of serializing it (see [this stackoverflow question](http://stackoverflow.com/questions/28267972/python-multiprocessing-locks)) which is what i did in a workaround in #1293. I now see your recommendation of using `multiprocessing.Manager().Lock()` works, so i'll use that instead of a global lock. Thanks!

The purpose of reporting this issue is mostly to make this problem visible and documented if it ever comes up, and maybe adding a explicit exception or warning to the code in the future.


Resolving patch:
diff --git a/dask/multiprocessing.py b/dask/multiprocessing.py
--- a/dask/multiprocessing.py
+++ b/dask/multiprocessing.py
@@ -1,25 +1,29 @@
 from __future__ import absolute_import, division, print_function
 
-from toolz import curry
-from .optimize import fuse, cull
 import multiprocessing
-from .async import get_async # TODO: get better get
+import pickle
+import sys
+
+from .async import get_async  # TODO: get better get
 from .context import _globals
-from sys import version
+from .optimize import fuse, cull
 
-if version < '3':
+import cloudpickle
+from toolz import curry
+
+
+if sys.version_info.major < 3:
     import copy_reg as copyreg
 else:
     import copyreg
 
+
 def _reduce_method_descriptor(m):
     return getattr, (m.__objclass__, m.__name__)
 
 # type(set.union) is used as a proxy to <class 'method_descriptor'>
 copyreg.pickle(type(set.union), _reduce_method_descriptor)
 
-import pickle
-import cloudpickle
 
 def _dumps(x):
     return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
@@ -63,8 +67,8 @@ def get(dsk, keys, num_workers=None, func_loads=None, func_dumps=None,
     queue = manager.Queue()
 
     apply_async = pickle_apply_async(pool.apply_async,
-                                          func_dumps=func_dumps,
-                                          func_loads=func_loads)
+                                     func_dumps=func_dumps,
+                                     func_loads=func_loads)
 
     # Optimize Dask
     dsk2, dependencies = cull(dsk, keys)
@@ -83,20 +87,37 @@ def get(dsk, keys, num_workers=None, func_loads=None, func_dumps=None,
     return result
 
 
-def apply_func(sfunc, sargs, skwds, loads=None):
+def apply_func(sfunc, may_fail, wont_fail, loads=None):
     loads = loads or _globals.get('loads') or _loads
     func = loads(sfunc)
-    args = loads(sargs)
-    kwds = loads(skwds)
-    return func(*args, **kwds)
+    key, queue, get_id, raise_on_exception = loads(wont_fail)
+    try:
+        task, data = loads(may_fail)
+    except Exception as e:
+        # Need a new reference for the exception, as `e` falls out of scope in
+        # python 3
+        exception = e
+        def serialization_failure():
+            raise exception
+        task = (serialization_failure,)
+        data = {}
+    return func(key, task, data, queue, get_id,
+                raise_on_exception=raise_on_exception)
 
 
 @curry
-def pickle_apply_async(apply_async, func, args=(), kwds={},
-                            func_loads=None, func_dumps=None):
+def pickle_apply_async(apply_async, func, args=(),
+                       func_loads=None, func_dumps=None):
+    # XXX: To deal with deserialization errors of tasks, this version of
+    # apply_async doesn't actually match that of `pool.apply_async`. It's
+    # customized to fit the signature of `dask.async.execute_task`, which is
+    # the only function ever actually passed as `func`. This is a bit of a
+    # hack, but it works pretty well. If the signature of `execute_task`
+    # changes, then this will need to be changed as well.
     dumps = func_dumps or _globals.get('func_dumps') or _dumps
+    key, task, data, queue, get_id, raise_on_exception = args
     sfunc = dumps(func)
-    sargs = dumps(args)
-    skwds = dumps(kwds)
+    may_fail = dumps((task, data))
+    wont_fail = dumps((key, queue, get_id, raise_on_exception))
     return apply_async(curry(apply_func, loads=func_loads),
-                       args=[sfunc, sargs, skwds])
+                       args=[sfunc, may_fail, wont_fail])


Test patch:
diff --git a/dask/tests/test_multiprocessing.py b/dask/tests/test_multiprocessing.py
--- a/dask/tests/test_multiprocessing.py
+++ b/dask/tests/test_multiprocessing.py
@@ -1,25 +1,13 @@
-import pytest
-
-from dask.multiprocessing import get, pickle_apply_async
-from dask.multiprocessing import _dumps, _loads
-from dask.context import set_options
 import multiprocessing
+import numpy as np
 import pickle
 from operator import add
-from dask.utils import raises
-import numpy as np
 
-inc = lambda x: x + 1
+from dask.context import set_options
+from dask.multiprocessing import get, _dumps, _loads
 
 
-def test_apply_lambda():
-    p = multiprocessing.Pool()
-    try:
-        result = pickle_apply_async(p.apply_async, lambda x: x + 1, args=[1])
-        assert isinstance(result, multiprocessing.pool.ApplyResult)
-        assert result.get() == 2
-    finally:
-        p.close()
+inc = lambda x: x + 1
 
 
 def test_pickle_globals():
@@ -40,7 +28,7 @@ def test_errors_propagate():
     dsk = {'x': (bad,)}
 
     try:
-        result = get(dsk, 'x')
+        get(dsk, 'x')
     except Exception as e:
         assert isinstance(e, ValueError)
         assert "12345" in str(e)
@@ -50,17 +38,47 @@ def make_bad_result():
     return lambda x: x + 1
 
 
-def test_unpicklable_results_genreate_errors():
+def test_unpicklable_results_generate_errors():
 
     dsk = {'x': (make_bad_result,)}
 
     try:
-        result = get(dsk, 'x')
+        get(dsk, 'x')
     except Exception as e:
         # can't use type because pickle / cPickle distinction
         assert type(e).__name__ in ('PicklingError', 'AttributeError')
 
 
+class NotUnpickleable(object):
+    def __getstate__(self):
+        return ()
+
+    def __setstate__(self, state):
+        raise ValueError("Can't unpickle me")
+
+
+def test_unpicklable_args_generate_errors():
+    a = NotUnpickleable()
+
+    def foo(a):
+        return 1
+
+    dsk = {'x': (foo, a)}
+
+    try:
+        get(dsk, 'x')
+    except Exception as e:
+        assert isinstance(e, ValueError)
+
+    dsk = {'x': (foo, 'a'),
+           'a': a}
+
+    try:
+        get(dsk, 'x')
+    except Exception as e:
+        assert isinstance(e, ValueError)
+
+
 def test_reuse_pool():
     pool = multiprocessing.Pool()
     with set_options(pool=pool):