Repository: dask/distributed

Issue:
Dangling stream when program exits
Hi,
  Not sure if this is a bug. There is no "close()" method on Client objects, but when my python program exits, I always see this.

distributed.comm.tcp - WARNING - Closing dangling stream in <TCP 'tcp://54.158.230.6:8786'>
distributed.comm.tcp - WARNING - Closing dangling stream in <TCP 'tcp://54.158.230.6:8786'>
Exception in thread Thread-6 (most likely raised during interpreter shutdown):
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
  File "/usr/lib/python2.7/threading.py", line 754, in run
  File "/home/radiant/radiantfind/python/radiant/local/lib/python2.7/site-packages/tornado/ioloop.py", line 904, in start
<type 'exceptions.AttributeError'>: 'NoneType' object has no attribute '_current'





Discussion:
What version are you running?

import distributed
distributed.__version__

On Fri, May 5, 2017 at 7:26 PM, jsonjunky <notifications@github.com> wrote:

> Hi,
> Not sure if this is a bug. There is no "close()" method on Client objects,
> but when my python program exits, I always see this.
>
> distributed.comm.tcp - WARNING - Closing dangling stream in <TCP 'tcp://
> 54.158.230.6:8786'>
> distributed.comm.tcp - WARNING - Closing dangling stream in <TCP 'tcp://
> 54.158.230.6:8786'>
> Exception in thread Thread-6 (most likely raised during interpreter
> shutdown):
> Traceback (most recent call last):
> File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
> File "/usr/lib/python2.7/threading.py", line 754, in run
> File "/home/radiant/radiantfind/python/radiant/local/lib/
> python2.7/site-packages/tornado/ioloop.py", line 904, in start
> <type 'exceptions.AttributeError'>: 'NoneType' object has no attribute
> '_current'
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/1064>, or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AASszJL1wOdYsAID2hPrXAbBePBNuqAGks5r27AogaJpZM4NShh9>
> .
>

>>> import distributed
>>> distributed.__version__
u'1.16.3'
>>> 
cc @pitrou any thoughts?

On Fri, May 5, 2017 at 9:21 PM, jsonjunky <notifications@github.com> wrote:

> import distributed
> distributed.*version*
> u'1.16.3'
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/distributed/issues/1064#issuecomment-299606709>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AASszOqN-3L8nBzHDxjLBn6YSrsPIYAtks5r28sCgaJpZM4NShh9>
> .
>

@jsonjunky, do you have a small reproducer?
Here is the smallest test. Using a remote dask scheduler. Do you want my pip freeze for this interpreter?

> from dask.distributed import Client
> client = Client("XX.211.XX.XX:8786")
> ctrl-d
distributed.comm.tcp - WARNING - Closing dangling stream in <TCP 'tcp://XX.211.XX.XX:8786'>
distributed.comm.tcp - WARNING - Closing dangling stream in <TCP 'tcp://XX.211.XX.XX:8786'>

Ok. According to my observations, the answer is the Client is not garbage-collected if you don't call Client.shutdown() explicitly (and/or use a context manager such as `with Client(...)`). The reason is probably that the client's event loop, which is running in a separate background thread, is holding references to the client, so even deleting all references from the client in your environment does not make the object vanish.

Ideally we would find a way to garbage-collected the Client (and close its thread and communications) when all user-visible references to it vanish, but due to the afore-mentioned problem, it is harder than it may seem at first.
That said, the warnings are specific to Python 2 and they are related to an issue in our compatibility code.

Resolving patch:
diff --git a/distributed/client.py b/distributed/client.py
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import atexit
 from collections import defaultdict, Iterator, Iterable
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures._base import DoneAndNotDoneFutures, CancelledError
@@ -19,6 +20,7 @@
 from threading import Thread, Lock
 import six
 import socket
+import weakref
 
 import dask
 from dask.base import tokenize, normalize_token, Base, collections_to_dsk
@@ -44,11 +46,20 @@
         tokey, log_errors, str_graph)
 from .versions import get_versions
 
+
 logger = logging.getLogger(__name__)
 
 _global_client = [None]
 
 
+def _get_global_client():
+    wr = _global_client[0]
+    return wr and wr()
+
+def _set_global_client(c):
+    _global_client[0] = weakref.ref(c) if c is not None else None
+
+
 class Future(WrappedKey):
     """ A remotely running computation
 
@@ -432,7 +443,7 @@ def start(self, **kwargs):
                 sleep(0.001)
         pc = PeriodicCallback(lambda: None, 1000, io_loop=self.loop)
         self.loop.add_callback(pc.start)
-        _global_client[0] = self
+        _set_global_client(self)
         sync(self.loop, self._start, **kwargs)
         self.status = 'running'
 
@@ -535,7 +546,7 @@ def _ensure_connected(self, timeout=5):
         bcomm.start(comm)
         self.scheduler_comm = bcomm
 
-        _global_client[0] = self
+        _set_global_client(self)
         self.status = 'running'
 
         for msg in self._pending_msg_buffer:
@@ -682,8 +693,8 @@ def _shutdown(self, fast=False):
                 with ignoring(AttributeError):
                     yield self.cluster._close()
             self.status = 'closed'
-            if _global_client[0] is self:
-                _global_client[0] = None
+            if _get_global_client() is self:
+                _set_global_client(None)
             if not fast:
                 with ignoring(TimeoutError):
                     yield [gen.with_timeout(timedelta(seconds=2), f)
@@ -2700,10 +2711,9 @@ def batches(self):
 
 def default_client(c=None):
     """ Return an client if exactly one has started """
+    c = c or _get_global_client()
     if c:
         return c
-    if _global_client[0]:
-        return _global_client[0]
     else:
         raise ValueError("No clients found\n"
                 "Start an client and point it to the scheduler address\n"
@@ -2760,8 +2770,21 @@ def temp_default_client(c):
         This is what default_client() will return within the with-block.
     """
     old_exec = default_client()
-    _global_client[0] = c
+    _set_global_client(c)
     try:
         yield
     finally:
-        _global_client[0] = old_exec
+        _set_global_client(old_exec)
+
+
+def _shutdown_global_client():
+    """
+    Force shutdown of global client.  This cleans up when a client
+    wasn't shutdown explicitly, e.g. interactive sessions.
+    """
+    c = _get_global_client()
+    if c is not None:
+        c.shutdown(timeout=2)
+
+
+atexit.register(_shutdown_global_client)
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -87,7 +87,7 @@ def isqueue(o):
     import itertools
     from weakref import ref
 
-    class finalize:
+    class finalize(object):
         """Class for finalization of weakrefable objects
 
         finalize(obj, func, *args, **kwargs) returns a callable finalizer


Test patch:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -30,7 +30,7 @@
 from distributed.comm import CommClosedError
 from distributed.utils_comm import WrappedKey
 from distributed.client import (Client, Future, _wait,
-        wait, _as_completed, as_completed, tokenize, _global_client,
+        wait, _as_completed, as_completed, tokenize, _get_global_client,
         default_client, _first_completed, ensure_default_get, futures_of,
         temp_default_client)
 from distributed.metrics import time
@@ -842,20 +842,20 @@ def test_get_releases_data(c, s, a, b):
 
 
 def test_global_clients(loop):
-    assert not _global_client[0]
+    assert _get_global_client() is None
     with pytest.raises(ValueError):
         default_client()
     with cluster() as (s, [a, b]):
         with Client(s['address'], loop=loop) as c:
-            assert _global_client == [c]
+            assert _get_global_client() is c
             assert default_client() is c
             with Client(s['address'], loop=loop) as f:
-                assert _global_client == [f]
+                assert _get_global_client() is f
                 assert default_client() is f
                 assert default_client(c) is c
                 assert default_client(f) is f
 
-    assert not _global_client[0]
+    assert _get_global_client() is None
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_compatibility.py b/distributed/tests/test_compatibility.py
--- a/distributed/tests/test_compatibility.py
+++ b/distributed/tests/test_compatibility.py
@@ -20,7 +20,10 @@ def cb(value):
         l.append(value)
 
     o = C()
-    finalize(o, cb, 1)
+    f = finalize(o, cb, 1)
+    assert f in f._select_for_exit()
+    f.atexit = False
+    assert f not in f._select_for_exit()
     assert not l
     del o
     assert l.pop() == 1