Repository: dask/distributed

Issue:
Deprecated bin/dask_mpi overwrites that from dask-mpi project
It appears there are two implementations of dask_mpi : The deprecated one in [distributed/cli/dask_mpi.py](https://github.com/dask/distributed/blob/master/distributed/cli/dask_mpi.py), and the newer [dask-mpi](https://github.com/dask/dask-mpi) project.

Somehow in Anaconda I managed to do the follow: (1) Install dask/distributed, (2) install dask-mpi, (3) reinstall dask/distributed. The problem is that step (3) overwrote the dask-mpi binary `bin/dask_mpi` with the deprecated one from dask/distributed.

This problem was resolved by removing and reinstalling the dask-mpi package. However, it took a while to figure out what's going on. Is there a way to avoid this problem of conflicting packages?


Discussion:

Resolving patch:
diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
deleted file mode 100644
--- a/distributed/cli/dask_mpi.py
+++ /dev/null
@@ -1,147 +0,0 @@
-from functools import partial
-
-import click
-from mpi4py import MPI
-from tornado.ioloop import IOLoop
-from tornado import gen
-from warnings import warn
-
-from distributed import Scheduler, Nanny, Worker
-from distributed.dashboard import BokehWorker
-from distributed.cli.utils import check_python_3
-from distributed.comm.addressing import uri_from_host_port
-from distributed.utils import get_ip_interface
-
-
-comm = MPI.COMM_WORLD
-rank = comm.Get_rank()
-loop = IOLoop()
-
-
-@click.command()
-@click.option(
-    "--scheduler-file",
-    type=str,
-    default="scheduler.json",
-    help="Filename to JSON encoded scheduler information. ",
-)
-@click.option(
-    "--interface", type=str, default=None, help="Network interface like 'eth0' or 'ib0'"
-)
-@click.option("--nthreads", type=int, default=0, help="Number of threads per worker.")
-@click.option(
-    "--memory-limit",
-    default="auto",
-    help="Number of bytes before spilling data to disk. "
-    "This can be an integer (nbytes) "
-    "float (fraction of total memory) "
-    "or 'auto'",
-)
-@click.option(
-    "--local-directory", default="", type=str, help="Directory to place worker files"
-)
-@click.option(
-    "--scheduler/--no-scheduler",
-    default=True,
-    help=(
-        "Whether or not to include a scheduler. "
-        "Use --no-scheduler to increase an existing dask cluster"
-    ),
-)
-@click.option(
-    "--nanny/--no-nanny",
-    default=True,
-    help="Start workers in nanny process for management",
-)
-@click.option(
-    "--bokeh-port", type=int, default=8787, help="Bokeh port for visual diagnostics"
-)
-@click.option(
-    "--bokeh-worker-port",
-    type=int,
-    default=8789,
-    help="Worker's Bokeh port for visual diagnostics",
-)
-@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
-@click.version_option()
-def main(
-    scheduler_file,
-    interface,
-    nthreads,
-    local_directory,
-    memory_limit,
-    scheduler,
-    bokeh_port,
-    bokeh_prefix,
-    nanny,
-    bokeh_worker_port,
-):
-    if interface:
-        host = get_ip_interface(interface)
-    else:
-        host = None
-
-    if rank == 0 and scheduler:
-        try:
-            from distributed.dashboard import BokehScheduler
-        except ImportError:
-            services = {}
-        else:
-            services = {
-                ("dashboard", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
-            }
-        scheduler = Scheduler(
-            scheduler_file=scheduler_file, loop=loop, services=services
-        )
-        addr = uri_from_host_port(host, None, 8786)
-        scheduler.start(addr)
-        try:
-            loop.start()
-            loop.close()
-        finally:
-            scheduler.stop()
-    else:
-        W = Nanny if nanny else Worker
-        worker = W(
-            scheduler_file=scheduler_file,
-            loop=loop,
-            name=rank if scheduler else None,
-            nthreads=nthreads,
-            local_dir=local_directory,
-            services={("dashboard", bokeh_worker_port): BokehWorker},
-            memory_limit=memory_limit,
-        )
-        addr = uri_from_host_port(host, None, 0)
-
-        @gen.coroutine
-        def run():
-            yield worker._start(addr)
-            while worker.status != "closed":
-                yield gen.sleep(0.2)
-
-        try:
-            loop.run_sync(run)
-            loop.close()
-        finally:
-            pass
-
-        @gen.coroutine
-        def close():
-            yield worker._close(timeout=2)
-
-        loop.run_sync(close)
-
-
-def go():
-    check_python_3()
-    warn(
-        "The dask-mpi command line utility in the `distributed` "
-        "package is deprecated.  "
-        "Please install the `dask-mpi` package instead. "
-        "More information is available at https://mpi.dask.org"
-    )
-    main()
-
-
-if __name__ == "__main__":
-    go()
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,6 @@
         dask-remote=distributed.cli.dask_remote:go
         dask-scheduler=distributed.cli.dask_scheduler:go
         dask-worker=distributed.cli.dask_worker:go
-        dask-mpi=distributed.cli.dask_mpi:go
       """,
     zip_safe=False,
 )


Test patch:
diff --git a/distributed/cli/tests/test_dask_mpi.py b/distributed/cli/tests/test_dask_mpi.py
deleted file mode 100644
--- a/distributed/cli/tests/test_dask_mpi.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from __future__ import print_function, division, absolute_import
-
-import subprocess
-from time import sleep
-
-import pytest
-
-pytest.importorskip("mpi4py")
-
-import requests
-from click.testing import CliRunner
-
-from distributed import Client
-from distributed.utils import tmpfile
-from distributed.metrics import time
-from distributed.utils_test import popen
-from distributed.utils_test import loop  # noqa: F401
-from distributed.cli.dask_remote import main
-
-
-@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
-def test_basic(loop, nanny):
-    with tmpfile() as fn:
-        with popen(
-            ["mpirun", "--np", "4", "dask-mpi", "--scheduler-file", fn, nanny],
-            stdin=subprocess.DEVNULL,
-        ):
-            with Client(scheduler_file=fn) as c:
-
-                start = time()
-                while len(c.scheduler_info()["workers"]) != 3:
-                    assert time() < start + 10
-                    sleep(0.2)
-
-                assert c.submit(lambda x: x + 1, 10, workers=1).result() == 11
-
-
-def test_no_scheduler(loop):
-    with tmpfile() as fn:
-        with popen(
-            ["mpirun", "--np", "2", "dask-mpi", "--scheduler-file", fn],
-            stdin=subprocess.DEVNULL,
-        ):
-            with Client(scheduler_file=fn) as c:
-
-                start = time()
-                while len(c.scheduler_info()["workers"]) != 1:
-                    assert time() < start + 10
-                    sleep(0.2)
-
-                assert c.submit(lambda x: x + 1, 10).result() == 11
-                with popen(
-                    [
-                        "mpirun",
-                        "--np",
-                        "1",
-                        "dask-mpi",
-                        "--scheduler-file",
-                        fn,
-                        "--no-scheduler",
-                    ]
-                ):
-
-                    start = time()
-                    while len(c.scheduler_info()["workers"]) != 2:
-                        assert time() < start + 10
-                        sleep(0.2)
-
-
-def test_bokeh(loop):
-    with tmpfile() as fn:
-        with popen(
-            [
-                "mpirun",
-                "--np",
-                "2",
-                "dask-mpi",
-                "--scheduler-file",
-                fn,
-                "--bokeh-port",
-                "59583",
-                "--bokeh-worker-port",
-                "59584",
-            ],
-            stdin=subprocess.DEVNULL,
-        ):
-
-            for port in [59853, 59584]:
-                start = time()
-                while True:
-                    try:
-                        response = requests.get("http://localhost:%d/status/" % port)
-                        assert response.ok
-                        break
-                    except Exception:
-                        sleep(0.1)
-                        assert time() < start + 20
-
-    with pytest.raises(Exception):
-        requests.get("http://localhost:59583/status/")
-
-
-def test_version_option():
-    runner = CliRunner()
-    result = runner.invoke(main, ["--version"])
-    assert result.exit_code == 0