Repository: dask/distributed

Issue:
Non-daemonic workers
Related to #2142, but the solution doesn't apply in my case. I have a use case for workers running in separate processes, but as non-daemons because the worker processes need to use multiprocessing. Here's an example:
```python
import torch
import torch.distributed as dist
import torchvision
import os
from distributed import Client, LocalCluster


def worker_fn(rank, world_size):
    print('worker', rank)

    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '8989'
    dist.init_process_group(
        backend=dist.Backend.NCCL,
        rank=rank,
        world_size=world_size,
    )
    print('initialized distributed', rank)

    if rank == 0:
        dataset = torchvision.datasets.MNIST(
            '../data/',
            train=True,
            download=True,
        )
    dist.barrier()
    if rank != 0:
        dataset = torchvision.datasets.MNIST(
            '../data/',
            train=True,
            download=False,
        )
    # load data, uses multiprocessing
    loader = torch.utils.data.DataLoader(
        dataset,
        sampler=torch.utils.data.distributed.DistributedSampler(
            dataset,
            rank=rank,
            num_replicas=world_size,
        ),
        num_workers=2,
    )
    print('constructed data loader', rank)

    # if cuda is available, initializes it as well
    assert torch.cuda.is_available()
    # do distributed training, but in this case it suffices to iterate
    for x, y in loader:
        pass


def main():
    world_size = 2
    cluster = LocalCluster(
        n_workers=world_size,
        processes=True,
        resources={
            'GPUS': 1,  # don't allow two tasks to run on the same worker
        },
    )
    cl = Client(cluster)
    futs = []
    for rank in range(world_size):
        futs.append(
            cl.submit(
                worker_fn,
                rank,
                world_size,
                resources={'GPUS': 1},
            ))

    for f in futs:
        f.result()


if __name__ == '__main__':
    main()
```
If `processes=True`, then we get an error about daemonic processes not being allowed to have children:
```
worker 0
worker 1
initialized distributed 1
initialized distributed 0
constructed data loader 0
constructed data loader 1
distributed.worker - WARNING -  Compute Failed
Function:  worker_fn
args:      (0, 2)
kwargs:    {}
Exception: AssertionError('daemonic processes are not allowed to have children',)

Traceback (most recent call last):
  File "scratch.py", line 152, in <module>
    main()
  File "scratch.py", line 148, in main
    f.result()
  File "/private/home/calebh/miniconda3/envs/fairtask2/lib/python3.6/site-packages/distributed/client.py", line 227, in result
    six.reraise(*result)
  File "/private/home/calebh/miniconda3/envs/fairtask2/lib/python3.6/site-packages/six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "scratch.py", line 123, in worker_fn
    for x, y in loader:
  File "/private/home/calebh/miniconda3/envs/fairtask2/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 193, in __iter__
    return _DataLoaderIter(self)
  File "/private/home/calebh/miniconda3/envs/fairtask2/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 469, in __init__
    w.start()
  File "/private/home/calebh/miniconda3/envs/fairtask2/lib/python3.6/multiprocessing/process.py", line 103, in start
    'daemonic processes are not allowed to have children'
AssertionError: daemonic processes are not allowed to have children
distributed.worker - WARNING -  Compute Failed
Function:  worker_fn
args:      (1, 2)
kwargs:    {}
Exception: AssertionError('daemonic processes are not allowed to have children',)
```

If `processes=False`, we get stuck at distributed initialization.




Discussion:
You might try setting up `dask-worker` processes yourself, perhaps with the `--no-nanny` flag.  If you come up with another easier way to handle this it would be great if you could share here.
The first solution that comes to mind is to not run workers as daemons, but I suspect there's a reason why they are. So why are workers run as daemons?
In particular, is there any reason why [this](https://github.com/dask/distributed/blob/master/distributed/nanny.py#L452) isn't a parameter to the `WorkerProcess` constructor?
I have no problem with making that configurable.  I might suggest a config option rather than piping it through all of the constructors.  

https://docs.dask.org/en/latest/configuration.html

Any interest in submitting a Pull Request with that change?
Yeah I can work on a PR

Resolving patch:
diff --git a/distributed/nanny.py b/distributed/nanny.py
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -455,7 +455,7 @@ def start(self):
                 env=self.env,
             ),
         )
-        self.process.daemon = True
+        self.process.daemon = dask.config.get("distributed.worker.daemon", default=True)
         self.process.set_exit_callback(self._on_exit)
         self.running = Event()
         self.stopped = Event()
diff --git a/distributed/process.py b/distributed/process.py
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -330,7 +330,7 @@ def daemon(self, value):
 @atexit.register
 def _cleanup_dangling():
     for proc in list(_dangling):
-        if proc.daemon and proc.is_alive():
+        if proc.is_alive():
             try:
                 logger.warning("reaping stray process %s" % (proc,))
                 proc.terminate()


Test patch:
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -5,6 +5,7 @@
 import os
 import random
 import sys
+import multiprocessing as mp
 
 import numpy as np
 
@@ -344,3 +345,37 @@ def test_data_types(c, s):
     r = yield c.run(lambda dask_worker: type(dask_worker.data))
     assert r[w.worker_address] == dict
     yield w.close()
+
+
+def _noop(x):
+    """Define here because closures aren't pickleable."""
+    pass
+
+
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    client=True,
+    Worker=Nanny,
+    config={"distributed.worker.daemon": False},
+)
+def test_mp_process_worker_no_daemon(c, s, a):
+    def multiprocessing_worker():
+        p = mp.Process(target=_noop, args=(None,))
+        p.start()
+        p.join()
+
+    yield c.submit(multiprocessing_worker)
+
+
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    client=True,
+    Worker=Nanny,
+    config={"distributed.worker.daemon": False},
+)
+def test_mp_pool_worker_no_daemon(c, s, a):
+    def pool_worker(world_size):
+        with mp.Pool(processes=world_size) as p:
+            p.map(_noop, range(world_size))
+
+    yield c.submit(pool_worker, 4)