Repository: dask/distributed

Issue:
test_move_unserializable_data hangs
While developing a test for #1345, I found a test which doesn't work on my machine. I'm on 8c14b2e.

Python version 3.5.1, Ubuntu 16.04, amd64.

The command I'm running, which reproduces it every time: `py.test-3 --showlocals --full-trace --verbose -k test_move_unserializable_data`. I believe I have let the test run for in excess of ten minutes before giving up on it.

---

<details>
<summary>Click to expand full output of py.test</summary>

```
distributed/deploy/tests/test_local.py::test_move_unserializable_data ^C

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! KeyboardInterrupt !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

config = <_pytest.config.Config object at 0x7f48266d7550>, doit = <function _main at 0x7f48265c40d0>

    def wrap_session(config, doit):
        """Skeleton command line program"""
        session = Session(config)
        session.exitstatus = EXIT_OK
        initstate = 0
        try:
            try:
                config._do_configure()
                initstate = 1
                config.hook.pytest_sessionstart(session=session)
                initstate = 2
>               session.exitstatus = doit(config, session) or 0

/usr/lib/python3/dist-packages/_pytest/main.py:90: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

config = <_pytest.config.Config object at 0x7f48266d7550>, session = <Session 'distributed'>

    def _main(config, session):
        """ default command line protocol for initialization, session,
        running tests and reporting. """
        config.hook.pytest_collection(session=session)
>       config.hook.pytest_runtestloop(session=session)

/usr/lib/python3/dist-packages/_pytest/main.py:121: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_HookCaller 'pytest_runtestloop'>
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'session': <Session 'distributed'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>, 'session': <Session 'distributed'>}

    def __call__(self, **kwargs):
        assert not self.is_historic()
>       return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:724: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.config.PytestPluginManager object at 0x7f4826655d68>, hook = <_HookCaller 'pytest_runtestloop'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'session': <Session 'distributed'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>, 'session': <Session 'distributed'>}

    def _hookexec(self, hook, methods, kwargs):
        # called from all hookcaller instances.
        # enable_tracing will set its own wrapping function at self._inner_hookexec
>       return self._inner_hookexec(hook, methods, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:338: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

hook = <_HookCaller 'pytest_runtestloop'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'session': <Session 'distributed'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>, 'session': <Session 'distributed'>}

    self._inner_hookexec = lambda hook, methods, kwargs: \
>       _MultiCall(methods, kwargs, hook.spec_opts).execute()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:333: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiCall 0 results, 0 meths, kwargs={'session': <Session 'distributed'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>

    def execute(self):
        all_kwargs = self.kwargs
        self.results = results = []
        firstresult = self.specopts.get("firstresult")
    
        while self.hook_impls:
            hook_impl = self.hook_impls.pop()
            args = [all_kwargs[argname] for argname in hook_impl.argnames]
            if hook_impl.hookwrapper:
                return _wrapped_call(hook_impl.function(*args), self.execute)
>           res = hook_impl.function(*args)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:596: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

session = <Session 'distributed'>

    def pytest_runtestloop(session):
        if session.config.option.collectonly:
            return True
    
        def getnextitem(i):
            # this is a function to avoid python2
            # keeping sys.exc_info set when calling into a test
            # python2 keeps sys.exc_info till the frame is left
            try:
                return session.items[i+1]
            except IndexError:
                return None
    
        for i, item in enumerate(session.items):
            nextitem = getnextitem(i)
>           item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)

/usr/lib/python3/dist-packages/_pytest/main.py:146: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_HookCaller 'pytest_runtest_protocol'>
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={......m': <Function 'test_move_unserializable_data'>}>, 'item': <Function 'test_move_unserializable_data'>, 'nextitem': None}

    def __call__(self, **kwargs):
        assert not self.is_historic()
>       return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:724: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.config.PytestPluginManager object at 0x7f4826655d68>, hook = <_HookCaller 'pytest_runtest_protocol'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={......m': <Function 'test_move_unserializable_data'>}>, 'item': <Function 'test_move_unserializable_data'>, 'nextitem': None}

    def _hookexec(self, hook, methods, kwargs):
        # called from all hookcaller instances.
        # enable_tracing will set its own wrapping function at self._inner_hookexec
>       return self._inner_hookexec(hook, methods, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:338: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

hook = <_HookCaller 'pytest_runtest_protocol'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={......m': <Function 'test_move_unserializable_data'>}>, 'item': <Function 'test_move_unserializable_data'>, 'nextitem': None}

    self._inner_hookexec = lambda hook, methods, kwargs: \
>       _MultiCall(methods, kwargs, hook.spec_opts).execute()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:333: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'nextitem': None, 'item': <Function 'test_move_unserializable_data'>}>

    def execute(self):
        all_kwargs = self.kwargs
        self.results = results = []
        firstresult = self.specopts.get("firstresult")
    
        while self.hook_impls:
            hook_impl = self.hook_impls.pop()
            args = [all_kwargs[argname] for argname in hook_impl.argnames]
            if hook_impl.hookwrapper:
>               return _wrapped_call(hook_impl.function(*args), self.execute)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:595: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

wrap_controller = <generator object pytest_runtest_protocol at 0x7f4825e57308>
func = <bound method _MultiCall.execute of <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'nextitem': None, 'item': <Function 'test_move_unserializable_data'>}>>

    def _wrapped_call(wrap_controller, func):
        """ Wrap calling to a function with a generator which needs to yield
        exactly once.  The yield point will trigger calling the wrapped function
        and return its _CallOutcome to the yield point.  The generator then needs
        to finish (raise StopIteration) in order for the wrapped call to complete.
        """
        try:
            next(wrap_controller)   # first yield
        except StopIteration:
            _raise_wrapfail(wrap_controller, "did not yield")
        call_outcome = _CallOutcome(func)
        try:
            wrap_controller.send(call_outcome)
            _raise_wrapfail(wrap_controller, "has second yield")
        except StopIteration:
            pass
>       return call_outcome.get_result()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:253: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.vendored_packages.pluggy._CallOutcome object at 0x7f480274a780>

    def get_result(self):
        if self.excinfo is None:
            return self.result
        else:
            ex = self.excinfo
            if _py3:
>               raise ex[1].with_traceback(ex[2])

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:278: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.vendored_packages.pluggy._CallOutcome object at 0x7f480274a780>
func = <bound method _MultiCall.execute of <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'nextitem': None, 'item': <Function 'test_move_unserializable_data'>}>>

    def __init__(self, func):
        try:
>           self.result = func()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:264: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'nextitem': None, 'item': <Function 'test_move_unserializable_data'>}>

    def execute(self):
        all_kwargs = self.kwargs
        self.results = results = []
        firstresult = self.specopts.get("firstresult")
    
        while self.hook_impls:
            hook_impl = self.hook_impls.pop()
            args = [all_kwargs[argname] for argname in hook_impl.argnames]
            if hook_impl.hookwrapper:
                return _wrapped_call(hook_impl.function(*args), self.execute)
>           res = hook_impl.function(*args)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:596: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <Function 'test_move_unserializable_data'>, nextitem = None

    def pytest_runtest_protocol(item, nextitem):
        item.ihook.pytest_runtest_logstart(
            nodeid=item.nodeid, location=item.location,
        )
>       runtestprotocol(item, nextitem=nextitem)

/usr/lib/python3/dist-packages/_pytest/runner.py:65: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <Function 'test_move_unserializable_data'>, log = True, nextitem = None

    def runtestprotocol(item, log=True, nextitem=None):
        hasrequest = hasattr(item, "_request")
        if hasrequest and not item._request:
            item._initrequest()
        rep = call_and_report(item, "setup", log)
        reports = [rep]
        if rep.passed:
>           reports.append(call_and_report(item, "call", log))

/usr/lib/python3/dist-packages/_pytest/runner.py:75: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <Function 'test_move_unserializable_data'>, when = 'call', log = True, kwds = {}

    def call_and_report(item, when, log=True, **kwds):
>       call = call_runtest_hook(item, when, **kwds)

/usr/lib/python3/dist-packages/_pytest/runner.py:119: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <Function 'test_move_unserializable_data'>, when = 'call', kwds = {}, hookname = 'pytest_runtest_call'

    def call_runtest_hook(item, when, **kwds):
        hookname = "pytest_runtest_" + when
        ihook = getattr(item.ihook, hookname)
>       return CallInfo(lambda: ihook(item=item, **kwds), when=when)

/usr/lib/python3/dist-packages/_pytest/runner.py:137: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <[AttributeError("'CallInfo' object has no attribute 'result'") raised in repr()] CallInfo object at 0x7f480274a358>
func = <function call_runtest_hook.<locals>.<lambda> at 0x7f4802bfc268>, when = 'call'

    def __init__(self, func, when):
        #: context of invocation: one of "setup", "call",
        #: "teardown", "memocollect"
        self.when = when
        self.start = time()
        try:
>           self.result = func()

/usr/lib/python3/dist-packages/_pytest/runner.py:149: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

>   return CallInfo(lambda: ihook(item=item, **kwds), when=when)

/usr/lib/python3/dist-packages/_pytest/runner.py:137: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_HookCaller 'pytest_runtest_call'>
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>, 'item': <Function 'test_move_unserializable_data'>}

    def __call__(self, **kwargs):
        assert not self.is_historic()
>       return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:724: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.config.PytestPluginManager object at 0x7f4826655d68>, hook = <_HookCaller 'pytest_runtest_call'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>, 'item': <Function 'test_move_unserializable_data'>}

    def _hookexec(self, hook, methods, kwargs):
        # called from all hookcaller instances.
        # enable_tracing will set its own wrapping function at self._inner_hookexec
>       return self._inner_hookexec(hook, methods, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:338: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

hook = <_HookCaller 'pytest_runtest_call'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>, 'item': <Function 'test_move_unserializable_data'>}

    self._inner_hookexec = lambda hook, methods, kwargs: \
>       _MultiCall(methods, kwargs, hook.spec_opts).execute()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:333: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>

    def execute(self):
        all_kwargs = self.kwargs
        self.results = results = []
        firstresult = self.specopts.get("firstresult")
    
        while self.hook_impls:
            hook_impl = self.hook_impls.pop()
            args = [all_kwargs[argname] for argname in hook_impl.argnames]
            if hook_impl.hookwrapper:
>               return _wrapped_call(hook_impl.function(*args), self.execute)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:595: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

wrap_controller = <generator object pytest_runtest_call at 0x7f48026e3e08>
func = <bound method _MultiCall.execute of <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>>

    def _wrapped_call(wrap_controller, func):
        """ Wrap calling to a function with a generator which needs to yield
        exactly once.  The yield point will trigger calling the wrapped function
        and return its _CallOutcome to the yield point.  The generator then needs
        to finish (raise StopIteration) in order for the wrapped call to complete.
        """
        try:
            next(wrap_controller)   # first yield
        except StopIteration:
            _raise_wrapfail(wrap_controller, "did not yield")
        call_outcome = _CallOutcome(func)
        try:
            wrap_controller.send(call_outcome)
            _raise_wrapfail(wrap_controller, "has second yield")
        except StopIteration:
            pass
>       return call_outcome.get_result()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:253: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.vendored_packages.pluggy._CallOutcome object at 0x7f480274a160>

    def get_result(self):
        if self.excinfo is None:
            return self.result
        else:
            ex = self.excinfo
            if _py3:
>               raise ex[1].with_traceback(ex[2])

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:278: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.vendored_packages.pluggy._CallOutcome object at 0x7f480274a160>
func = <bound method _MultiCall.execute of <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>>

    def __init__(self, func):
        try:
>           self.result = func()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:264: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiCall 0 results, 0 meths, kwargs={'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>, 'item': <Function 'test_move_unserializable_data'>}>

    def execute(self):
        all_kwargs = self.kwargs
        self.results = results = []
        firstresult = self.specopts.get("firstresult")
    
        while self.hook_impls:
            hook_impl = self.hook_impls.pop()
            args = [all_kwargs[argname] for argname in hook_impl.argnames]
            if hook_impl.hookwrapper:
                return _wrapped_call(hook_impl.function(*args), self.execute)
>           res = hook_impl.function(*args)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:596: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

item = <Function 'test_move_unserializable_data'>

    def pytest_runtest_call(item):
        try:
>           item.runtest()

/usr/lib/python3/dist-packages/_pytest/runner.py:90: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Function 'test_move_unserializable_data'>

    def runtest(self):
        """ execute the underlying test function. """
>       self.ihook.pytest_pyfunc_call(pyfuncitem=self)

/usr/lib/python3/dist-packages/_pytest/python.py:1406: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_HookCaller 'pytest_pyfunc_call'>
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'pyfuncitem': <Function 'test_move_unserializable_data'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>, 'pyfuncitem': <Function 'test_move_unserializable_data'>}

    def __call__(self, **kwargs):
        assert not self.is_historic()
>       return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:724: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_pytest.config.PytestPluginManager object at 0x7f4826655d68>, hook = <_HookCaller 'pytest_pyfunc_call'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'pyfuncitem': <Function 'test_move_unserializable_data'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>, 'pyfuncitem': <Function 'test_move_unserializable_data'>}

    def _hookexec(self, hook, methods, kwargs):
        # called from all hookcaller instances.
        # enable_tracing will set its own wrapping function at self._inner_hookexec
>       return self._inner_hookexec(hook, methods, kwargs)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:338: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

hook = <_HookCaller 'pytest_pyfunc_call'>, methods = []
kwargs = {'__multicall__': <_MultiCall 0 results, 0 meths, kwargs={'pyfuncitem': <Function 'test_move_unserializable_data'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>, 'pyfuncitem': <Function 'test_move_unserializable_data'>}

    self._inner_hookexec = lambda hook, methods, kwargs: \
>       _MultiCall(methods, kwargs, hook.spec_opts).execute()

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:333: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_MultiCall 0 results, 0 meths, kwargs={'pyfuncitem': <Function 'test_move_unserializable_data'>, '__multicall__': <_MultiCall 0 results, 0 meths, kwargs={...}>}>

    def execute(self):
        all_kwargs = self.kwargs
        self.results = results = []
        firstresult = self.specopts.get("firstresult")
    
        while self.hook_impls:
            hook_impl = self.hook_impls.pop()
            args = [all_kwargs[argname] for argname in hook_impl.argnames]
            if hook_impl.hookwrapper:
                return _wrapped_call(hook_impl.function(*args), self.execute)
>           res = hook_impl.function(*args)

/usr/lib/python3/dist-packages/_pytest/vendored_packages/pluggy.py:596: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

pyfuncitem = <Function 'test_move_unserializable_data'>

    @pytest.hookimpl(trylast=True)
    def pytest_pyfunc_call(pyfuncitem):
        testfunction = pyfuncitem.obj
        if pyfuncitem._isyieldedfunction():
            testfunction(*pyfuncitem._args)
        else:
            funcargs = pyfuncitem.funcargs
            testargs = {}
            for arg in pyfuncitem._fixtureinfo.argnames:
                testargs[arg] = funcargs[arg]
>           testfunction(**testargs)

/usr/lib/python3/dist-packages/_pytest/python.py:286: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    def test_move_unserializable_data():
        """
        Test that unserializable data is still fine to transfer over inproc
        transports.
        """
        with LocalCluster(processes=False, silence_logs=False,
                          diagnostics_port=None) as cluster:
            assert cluster.scheduler_address.startswith('inproc://')
            assert cluster.workers[0].address.startswith('inproc://')
            with Client(cluster) as client:
                lock = Lock()
                [x] = client.scatter([lock])
                y = client.submit(lambda x: x, x)
>               assert y.result() is lock

distributed/deploy/tests/test_local.py:86: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Future: status: cancelled, key: <lambda>-c77d9b652c39b9a35d1ef82cd3759bc6>, timeout = None

    def result(self, timeout=None):
        """ Wait until computation completes, gather result to local process.
    
            If *timeout* seconds are elapsed before returning, a TimeoutError
            is raised.
            """
        if self.client.asynchronous:
            result = self._result
            if timeout:
                result = gen.with_timeout(timedelta(seconds=timeout), result)
            return result
        result = sync(self.client.loop,
>                     self._result, raiseit=False, callback_timeout=timeout)

distributed/client.py:155: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

loop = <tornado.platform.epoll.EPollIOLoop object at 0x7f48027797b8>
func = <bound method Future._result of <Future: status: cancelled, key: <lambda>-c77d9b652c39b9a35d1ef82cd3759bc6>>, args = (), kwargs = {'raiseit': False}
f = <function sync.<locals>.f at 0x7f480273abf8>

    def sync(loop, func, *args, **kwargs):
        """
        Run coroutine in loop running in separate thread.
        """
        timeout = kwargs.pop('callback_timeout', None)
        def make_coro():
            coro = gen.maybe_future(func(*args, **kwargs))
            if timeout is None:
                return coro
            else:
                return gen.with_timeout(timedelta(seconds=timeout), coro)
    
        if not loop._running:
            try:
                return loop.run_sync(make_coro)
            except RuntimeError:  # loop already running
                pass
    
        e = threading.Event()
        main_tid = get_thread_identity()
        result = [None]
        error = [False]
    
        @gen.coroutine
        def f():
            try:
                if main_tid == get_thread_identity():
                    raise RuntimeError("sync() called from thread of running loop")
                yield gen.moment
                thread_state.asynchronous = True
                result[0] = yield make_coro()
            except Exception as exc:
                logger.exception(exc)
                error[0] = sys.exc_info()
            finally:
                thread_state.asynchronous = False
                e.set()
    
        loop.add_callback(f)
        while not e.is_set():
>           e.wait(1000000)

distributed/utils.py:237: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <threading.Event object at 0x7f4802f0e978>, timeout = 1000000

    def wait(self, timeout=None):
        """Block until the internal flag is true.
    
            If the internal flag is true on entry, return immediately. Otherwise,
            block until another thread calls set() to set the flag to true, or until
            the optional timeout occurs.
    
            When the timeout argument is present and not None, it should be a
            floating point number specifying a timeout for the operation in seconds
            (or fractions thereof).
    
            This method returns the internal flag on exit, so it will always return
            True except if a timeout is given and the operation times out.
    
            """
        with self._cond:
            signaled = self._flag
            if not signaled:
>               signaled = self._cond.wait(timeout)

/usr/lib/python3.5/threading.py:549: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Condition(<unlocked _thread.lock object at 0x7f4802701440>, 0)>, timeout = 1000000

    def wait(self, timeout=None):
        """Wait until notified or until a timeout occurs.
    
            If the calling thread has not acquired the lock when this method is
            called, a RuntimeError is raised.
    
            This method releases the underlying lock, and then blocks until it is
            awakened by a notify() or notify_all() call for the same condition
            variable in another thread, or until the optional timeout occurs. Once
            awakened or timed out, it re-acquires the lock and returns.
    
            When the timeout argument is present and not None, it should be a
            floating point number specifying a timeout for the operation in seconds
            (or fractions thereof).
    
            When the underlying lock is an RLock, it is not released using its
            release() method, since this may not actually unlock the lock when it
            was acquired multiple times recursively. Instead, an internal interface
            of the RLock class is used, which really unlocks it even when it has
            been recursively acquired several times. Another internal interface is
            then used to restore the recursion level when the lock is reacquired.
    
            """
        if not self._is_owned():
            raise RuntimeError("cannot wait on un-acquired lock")
        waiter = _allocate_lock()
        waiter.acquire()
        self._waiters.append(waiter)
        saved_state = self._release_save()
        gotit = False
        try:    # restore state no matter what (e.g., KeyboardInterrupt)
            if timeout is None:
                waiter.acquire()
                gotit = True
            else:
                if timeout > 0:
>                   gotit = waiter.acquire(True, timeout)
E                   KeyboardInterrupt

/usr/lib/python3.5/threading.py:297: KeyboardInterrupt
```

</details>

---

Running pytest with `--capture=no`, the following output is seen:

```
distributed/deploy/tests/test_local.py::test_move_unserializable_data distributed.scheduler - INFO -   Scheduler at: inproc://192.168.1.2/8904/1
distributed.worker - INFO -       Start worker at: inproc://192.168.1.2/8904/2
distributed.worker - INFO - Waiting to connect to: inproc://192.168.1.2/8904/1
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -               Threads:                          8
distributed.worker - INFO -                Memory:                    9.93 GB
distributed.worker - INFO -       Local Directory:       /tmp/worker-b0xnishm
distributed.worker - INFO - -------------------------------------------------
distributed.scheduler - INFO - Register inproc://192.168.1.2/8904/2
distributed.scheduler - INFO - Starting worker compute stream, inproc://192.168.1.2/8904/2
distributed.worker - INFO -         Registered to:      inproc://192.168.1.2/8904/1
distributed.worker - INFO - -------------------------------------------------
distributed.scheduler - INFO - Receive client connection: Client-80bab9e2-871a-11e7-a2c8-50e5495316a8
distributed.protocol.pickle - INFO - Failed to deserialize b"\x80\x04\x95o\x01\x00\x00\x00\x00\x00\x00\x8c\x17cloudpickle.cloudpickle\x94\x8c\x19_rehydrate_skeleton_class\x94\x93\x94(h\x00\x8c\r_builtin_type\x94\x93\x94\x8c\tClassType\x94\x85\x94R\x94\x8c\x04lock\x94\x8c\x08builtins\x94\x8c\x06object\x94\x93\x94\x85\x94}\x94\x8c\x07__doc__\x94Ns\x87\x94R\x94}\x94(\x8c\x0crelease_lock\x94\x8c\x08builtins\x94\x8c\x07getattr\x94\x93\x94h\x10h\x12\x86\x94R\x94\x8c\t__enter__\x94h\x15h\x10h\x18\x86\x94R\x94\x8c\x08__repr__\x94h\x15h\x10h\x1b\x86\x94R\x94\x8c\x0blocked_lock\x94h\x15h\x10h\x1e\x86\x94R\x94\x8c\x0cacquire_lock\x94h\x15h\x10h!\x86\x94R\x94\x8c\x07release\x94h\x15h\x10h$\x86\x94R\x94\x8c\x08__exit__\x94h\x15h\x10h'\x86\x94R\x94\x8c\x07acquire\x94h\x15h\x10h*\x86\x94R\x94\x8c\x06locked\x94h\x15h\x10h-\x86\x94R\x94utR."
Traceback (most recent call last):
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/protocol/pickle.py", line 59, in loads
    return pickle.loads(x)
AttributeError: type object 'lock' has no attribute 'release_lock'
distributed.client - ERROR - type object 'lock' has no attribute 'release_lock'
Traceback (most recent call last):
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/client.py", line 832, in _handle_report
    handler(**msg)
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/client.py", line 842, in _handle_key_in_memory
    type = loads(type)
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/protocol/pickle.py", line 59, in loads
    return pickle.loads(x)
AttributeError: type object 'lock' has no attribute 'release_lock'
```

---

I note that the exception branch of [`sync()` does not do anything with the event](https://github.com/dask/distributed/blob/8c14b2e16999a5ec78ca9c965c20d3a03e3614c4/distributed/utils.py#L228-L230), so line 237 is left [waiting a long time](https://github.com/dask/distributed/blob/8c14b2e16999a5ec78ca9c965c20d3a03e3614c4/distributed/utils.py#L237).




Discussion:
Its odd that you enter pickle at all in that test, it's using the inproc protocol and so shouldn't be serializing anything.

Regarding `sync` we call `e.set` in the finally branch of the `try-except-finally` statement
> Regarding sync we call e.set in the finally branch of the try-except-finally statement

Oh, so you do - misread it, sorry.

These tests also hang:

```
distributed/tests/test_queues.py::test_picklability <- distributed/utils_test.py
distributed/tests/test_stress.py::test_stress_steal <- distributed/utils_test.py 
```

There are a number of failing tests, too.

Interrupting `distributed/tests/test_stress.py::test_stress_steal`, I see a huge number of identical stack traces on stderr:

```
tornado.application - ERROR - Exception in callback (<socket.socket fd=59, family=AddressFamily.AF_INET, type=2049, proto=6, laddr=('127.0.0.1', 40390)>, <function wrap.<locals>.null_wrapper at 0x7fb864585ae8>)
Traceback (most recent call last):
  File "/home/pwaller/.local/lib/python3.5/site-packages/tornado/ioloop.py", line 888, in start
    handler_func(fd_obj, events)
  File "/home/pwaller/.local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper
    return fn(*args, **kwargs)
  File "/home/pwaller/.local/lib/python3.5/site-packages/tornado/netutil.py", line 264, in accept_handler
  File "/usr/lib/python3.5/socket.py", line 195, in accept
OSError: [Errno 24] Too many open files
```

Is it expected to require more than 1024 files (the default on my Ubuntu system) to run the tests? If so, it might be worth mentioning it in [the docs](https://distributed.readthedocs.io/en/latest/develop.html#test). Or even better if possible, to prevent them from running if the limit is lower than needed to run the tests. I note that the CI runs with 30,000. I'm now setting the tests running with this ulimit to see if it makes a difference.
Are you on master or latest release?
Master @ 8c14b2e.
That's surprising.  Do you get many warnings about leaking file descriptors when tests finish?  If so are there particular tests that leak a particularly large number?  Or the large number of file descriptors just within this one test?
Also can you provide a dump of your Python environment?  
> Also can you provide a dump of your Python environment?

Can you clarify what you want "dumped" and how to dump it?
A list of all software packages that you have installed so that I can try to reproduce your error.  The tests above work well on my environment and on the environment that we use for testing on travis-ci and appveyor.

Typically people provide environments with the `conda list` (strongly preferred) or `pip list` commands.
I reproduced this with an empty virtualenv with `--no-site-packages`, here's what I did to install it, and the list of packages I have installed as a result. 

```
pip3 install -e /home/pwaller/.local/src/github.com/cloudpipe/cloudpickle \
  -e /home/pwaller/.local/src/github.com/dask/dask'[complete]' \
  -e /home/pwaller/.local/src/github.com/dask/distributed
pip3 install -r requirements.txt -r dev-requirements.txt
```

```
(venv) ~/.local/src/github.com/dask/distributed$ pip3 list --format=columns
Package          Version                  Location                                                 
---------------- ------------------------ ---------------------------------------------------------
bkcharts         0.2                      
bokeh            0.12.6                   
boto3            1.4.6                    
botocore         1.6.4                    
certifi          2017.7.27.1              
chardet          3.0.4                    
click            6.7                      
cloudpickle      0.3.1                    /home/pwaller/.local/src/github.com/cloudpipe/cloudpickle
dask             0.15.1+33.gaafeedc       /home/pwaller/.local/src/github.com/dask/dask            
decorator        4.1.2                    
distributed      1.18.0+46.g8c14b2e.dirty /home/pwaller/.local/src/github.com/dask/distributed     
docutils         0.14                     
HeapDict         1.0.0                    
idna             2.6                      
ipykernel        4.6.1                    
ipython          6.1.0                    
ipython-genutils 0.2.0                    
jedi             0.10.2                   
Jinja2           2.9.6                    
jmespath         0.9.3                    
joblib           0.11                     
jupyter-client   5.1.0                    
jupyter-core     4.3.0                    
locket           0.2.0                    
MarkupSafe       1.0                      
mock             2.0.0                    
msgpack-python   0.4.8                    
numpy            1.13.1                   
pandas           0.20.3                   
partd            0.3.8                    
pbr              3.1.1                    
pexpect          4.2.1                    
pickleshare      0.7.4                    
pip              9.0.1                    
pkg-resources    0.0.0                    
prompt-toolkit   1.0.15                   
psutil           5.2.2                    
ptyprocess       0.5.2                    
py               1.4.34                   
Pygments         2.2.0                    
pytest           3.2.1                    
python-dateutil  2.6.1                    
pytz             2017.2                   
PyYAML           3.12                     
pyzmq            16.0.2                   
requests         2.18.4                   
s3fs             0.1.2                    
s3transfer       0.1.10                   
setuptools       36.2.7                   
simplegeneric    0.8.1                    
six              1.10.0                   
sortedcontainers 1.5.7                    
tblib            1.3.2                    
toolz            0.8.2                    
tornado          4.5.1                    
traitlets        4.3.2                    
urllib3          1.22                     
wcwidth          0.1.7                    
wheel            0.30.0a0                 
zict             0.1.2
```
I have discovered that I had master of cloudpickle checked out, and that this is the problem.

https://github.com/cloudpipe/cloudpickle/commit/aec80d21ddff84cf2a83dce3cb5921a9f58ffd05 is the first bad commit.
The stderr with master of cloudpickle:

```
distributed.worker - INFO -       Start worker at: inproc://192.168.1.2/31890/2
distributed.worker - INFO - Waiting to connect to: inproc://192.168.1.2/31890/1
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -               Threads:                          8
distributed.worker - INFO -                Memory:                    9.93 GB
distributed.worker - INFO -       Local Directory:       /tmp/worker-6f2qgljy
distributed.worker - INFO - -------------------------------------------------
distributed.scheduler - INFO - Register inproc://192.168.1.2/31890/2
distributed.scheduler - INFO - Starting worker compute stream, inproc://192.168.1.2/31890/2
distributed.worker - INFO -         Registered to:     inproc://192.168.1.2/31890/1
distributed.worker - INFO - -------------------------------------------------
distributed.scheduler - INFO - Receive client connection: Client-608f4414-8766-11e7-bc92-50e5495316a8
distributed.protocol.pickle - INFO - Failed to deserialize b"\x80\x04\x95o\x01\x00\x00\x00\x00\x00\x00\x8c\x17cloudpickle.cloudpickle\x94\x8c\x19_rehydrate_skeleton_class\x94\x93\x94(h\x00\x8c\r_builtin_type\x94\x93\x94\x8c\tClassType\x94\x85\x94R\x94\x8c\x04lock\x94\x8c\x08builtins\x94\x8c\x06object\x94\x93\x94\x85\x94}\x94\x8c\x07__doc__\x94Ns\x87\x94R\x94}\x94(\x8c\x07release\x94\x8c\x08builtins\x94\x8c\x07getattr\x94\x93\x94h\x10h\x12\x86\x94R\x94\x8c\x06locked\x94h\x15h\x10h\x18\x86\x94R\x94\x8c\x07acquire\x94h\x15h\x10h\x1b\x86\x94R\x94\x8c\t__enter__\x94h\x15h\x10h\x1e\x86\x94R\x94\x8c\x08__repr__\x94h\x15h\x10h!\x86\x94R\x94\x8c\x0blocked_lock\x94h\x15h\x10h$\x86\x94R\x94\x8c\x0cacquire_lock\x94h\x15h\x10h'\x86\x94R\x94\x8c\x0crelease_lock\x94h\x15h\x10h*\x86\x94R\x94\x8c\x08__exit__\x94h\x15h\x10h-\x86\x94R\x94utR."
Traceback (most recent call last):
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/protocol/pickle.py", line 59, in loads
    return pickle.loads(x)
AttributeError: type object 'lock' has no attribute 'release'
distributed.client - ERROR - type object 'lock' has no attribute 'release'
Traceback (most recent call last):
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/client.py", line 832, in _handle_report
    handler(**msg)
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/client.py", line 842, in _handle_key_in_memory
    type = loads(type)
  File "/home/pwaller/.local/src/github.com/dask/distributed/distributed/protocol/pickle.py", line 59, in loads
    return pickle.loads(x)
AttributeError: type object 'lock' has no attribute 'release'
^Cdistributed.scheduler - INFO - Connection to client Client-608f4414-8766-11e7-bc92-50e5495316a8 broken
distributed.scheduler - INFO - Remove client Client-608f4414-8766-11e7-bc92-50e5495316a8
distributed.scheduler - INFO - Close client connection: Client-608f4414-8766-11e7-bc92-50e5495316a8
distributed.worker - INFO - Stopping worker at inproc://192.168.1.2/31890/2
distributed.scheduler - INFO - Remove worker inproc://192.168.1.2/31890/2
distributed.scheduler - INFO - Lost all workers
distributed.scheduler - INFO - Scheduler closing...
distributed.scheduler - INFO - Scheduler closing all comms
```

And with 0.3.1 checked out:

```
distributed.worker - INFO -       Start worker at: inproc://192.168.1.2/31709/2
distributed.worker - INFO - Waiting to connect to: inproc://192.168.1.2/31709/1
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -               Threads:                          8
distributed.worker - INFO -                Memory:                    9.93 GB
distributed.worker - INFO -       Local Directory:       /tmp/worker-5jzkvq2l
distributed.worker - INFO - -------------------------------------------------
distributed.scheduler - INFO - Register inproc://192.168.1.2/31709/2
distributed.scheduler - INFO - Starting worker compute stream, inproc://192.168.1.2/31709/2
distributed.worker - INFO -         Registered to:     inproc://192.168.1.2/31709/1
distributed.worker - INFO - -------------------------------------------------
distributed.scheduler - INFO - Receive client connection: Client-4fa7dc34-8766-11e7-bbdd-50e5495316a8
distributed.protocol.pickle - INFO - Failed to serialize <class '_thread.lock'>. Exception: Could not pickle object as excessively deep recursion required.
distributed.scheduler - INFO - Connection to client Client-4fa7dc34-8766-11e7-bbdd-50e5495316a8 broken
distributed.scheduler - INFO - Remove client Client-4fa7dc34-8766-11e7-bbdd-50e5495316a8
distributed.scheduler - INFO - Close client connection: Client-4fa7dc34-8766-11e7-bbdd-50e5495316a8
distributed.worker - INFO - Stopping worker at inproc://192.168.1.2/31709/2
distributed.scheduler - INFO - Remove worker inproc://192.168.1.2/31709/2
distributed.scheduler - INFO - Lost all workers
distributed.scheduler - INFO - Scheduler closing...
distributed.scheduler - INFO - Scheduler closing all comms
```

Note in particular this ~error~ (edit: 'INFO') message looks interesting: `distributed.protocol.pickle - INFO - Failed to serialize <class '_thread.lock'>. Exception: Could not pickle object as excessively deep recursion required.`
This has now manifested itself in the tests on travis-ci.  
Raised upstream in https://github.com/cloudpipe/cloudpickle/issues/117

Resolving patch:
diff --git a/distributed/client.py b/distributed/client.py
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -839,7 +839,10 @@ def _handle_key_in_memory(self, key=None, type=None, workers=None):
         state = self.futures.get(key)
         if state is not None:
             if type and not state.type:  # Type exists and not yet set
-                type = loads(type)
+                try:
+                    type = loads(type)
+                except Exception:
+                    type = None
                 # Here, `type` may be a str if actual type failed
                 # serializing in Worker
             else:


Test patch:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -81,7 +81,7 @@ def test_move_unserializable_data():
         assert cluster.workers[0].address.startswith('inproc://')
         with Client(cluster) as client:
             lock = Lock()
-            [x] = client.scatter([lock])
+            x = client.scatter(lock)
             y = client.submit(lambda x: x, x)
             assert y.result() is lock