Repository: dask/distributed

Issue:
IOLoop instantiated on import of distributed breaks tornado forking
I've noticed that importing distributed seems to create an `IOLoop` which can break further code that tries to fork multiple processes, assuming the main process does not have an `IOLoop`. The workaround is to move around the imports, but that can get messy, and I don't think an import should be instantiating this to begin with.

**Sample code run on Linux:**
```python
from tornado.ioloop import IOLoop
from tornado.netutil import bind_sockets
from tornado.process import fork_processes
from tornado.tcpserver import TCPServer

print(f"IOLoop before import: {IOLoop.current(instance=False)}")
import distributed
print(f"IOLoop after import: {IOLoop.current(instance=False)}")

sockets = bind_sockets(8888)
fork_processes(2, max_restarts=1)
server = TCPServer()
server.add_sockets(sockets)
IOLoop.current().start()
```

**Error message:**
```
IOLoop before import: None
IOLoop after import: <tornado.platform.asyncio.AsyncIOMainLoop object at 0x7fffef352490>
Traceback (most recent call last):
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/asyncio/selector_events.py", line 259, in _add_reader
    key = self._selector.get_key(fd)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/selectors.py", line 192, in get_key
    raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '11 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "distributed_test.py", line 13, in <module>
    server.add_sockets(sockets)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/tcpserver.py", line 165, in add_sockets
    self._handlers[sock.fileno()] = add_accept_handler(
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/netutil.py", line 282, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/platform/asyncio.py", line 150, in add_handler
    self.selector_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/asyncio/selector_events.py", line 332, in add_reader
    return self._add_reader(fd, callback, *args)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/asyncio/selector_events.py", line 261, in _add_reader
    self._selector.register(fd, selectors.EVENT_READ,
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/selectors.py", line 359, in register
    self._selector.register(key.fd, poller_events)
FileExistsError: [Errno 17] File exists
Traceback (most recent call last):
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/asyncio/selector_events.py", line 259, in _add_reader
    key = self._selector.get_key(fd)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/selectors.py", line 192, in get_key
    raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '11 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "distributed_test.py", line 13, in <module>
    server.add_sockets(sockets)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/tcpserver.py", line 165, in add_sockets
    self._handlers[sock.fileno()] = add_accept_handler(
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/netutil.py", line 282, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/platform/asyncio.py", line 150, in add_handler
    self.selector_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/asyncio/selector_events.py", line 332, in add_reader
    return self._add_reader(fd, callback, *args)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/asyncio/selector_events.py", line 261, in _add_reader
    self._selector.register(fd, selectors.EVENT_READ,
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/selectors.py", line 359, in register
    self._selector.register(key.fd, poller_events)
FileExistsError: [Errno 17] File exists
Traceback (most recent call last):
  File "distributed_test.py", line 11, in <module>
    fork_processes(2, max_restarts=1)
  File "/home/user/miniconda3/envs/distributed_test/lib/python3.8/site-packages/tornado/process.py", line 167, in fork_processes
    raise RuntimeError("Too many child restarts, giving up")
RuntimeError: Too many child restarts, giving up
```

Debugging through the import, I came across the below where the `ExecutorResolver` attached to the `BaseTCPConnector` class implicitly creates an `IOLoop` on initialization.
https://github.com/dask/distributed/blob/cbb88bd43288a7e25a03e11f17e1c9a2f5e6f31d/distributed/comm/tcp.py#L351-L353

**Tornado code:**
https://github.com/tornadoweb/tornado/blob/b4e39e52cd27d6b3b324a399dff046f71545c4a5/tornado/netutil.py#L422



Discussion:

Resolving patch:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -386,8 +386,20 @@ def _check_encryption(self, address, connection_args):
 
 class BaseTCPConnector(Connector, RequireEncryptionMixin):
     _executor = ThreadPoolExecutor(2, thread_name_prefix="TCP-Executor")
-    _resolver = netutil.ExecutorResolver(close_executor=False, executor=_executor)
-    client = TCPClient(resolver=_resolver)
+
+    @property
+    def client(self):
+        # The `TCPClient` is cached on the class itself to avoid creating
+        # excess `ThreadPoolExecutor`s. We delay creation until inside an async
+        # function to avoid accessing an IOLoop from a context where a backing
+        # event loop doesn't exist.
+        cls = type(self)
+        if not hasattr(type(self), "_client"):
+            resolver = netutil.ExecutorResolver(
+                close_executor=False, executor=cls._executor
+            )
+            cls._client = TCPClient(resolver=resolver)
+        return cls._client
 
     async def connect(self, address, deserialize=True, **connection_args):
         self._check_encryption(address, connection_args)


Test patch:
diff --git a/distributed/tests/test_imports.py b/distributed/tests/test_imports.py
new file mode 100644
--- /dev/null
+++ b/distributed/tests/test_imports.py
@@ -0,0 +1,36 @@
+import subprocess
+import sys
+
+script = """
+import threading
+import traceback
+
+import_succeeded = False
+
+def import_dask_distributed():
+    try:
+        import dask.distributed  # noqa
+    except Exception:
+        # Print the exception for debugging
+        traceback.print_exc()
+    else:
+        global import_succeeded
+        import_succeeded = True
+
+thread = threading.Thread(target=import_dask_distributed, daemon=True)
+thread.start()
+thread.join()
+
+assert import_succeeded
+
+# Now try to use distributed from the main thread, and check that it works
+from dask.distributed import Client
+with Client(processes=False) as client:
+    res = client.submit(lambda x: x + 1, 1).result()
+    assert res == 2
+"""
+
+
+def test_can_import_distributed_in_background_thread():
+    """See https://github.com/dask/distributed/issues/5587"""
+    subprocess.check_call([sys.executable, "-c", script])