Repository: dask/distributed

Issue:
Start multiple workers with single ssh connection via SSHCluster
In the release 2021.10.0 the pull request https://github.com/dask/distributed/pull/5191 is included, where the SSHCluster uses now the Nanny class instead of the cli.worker command.
This broke my workflow.

I use the SSHCluster to deploy a cluster on a HPC-Cluster. I allocate several nodes with 64cores each. My approach was to start a wokers with the nprocs command. So the length of the hosts list equals the number of nodes (+1 for the scheduler) and i started 64 processes there.
Now I had to enlarge the number of hosts to create the same number of workers, but for every worker a ssh connection has to be established to start them. Now the problem is, that the number of ssh connections is limited to ~50 and starting the high number of workers failed.
Are there any suggestions how start multiple workers with a single ssh connection?

I will try to create a minimal example next week.


Discussion:

Resolving patch:
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -9,7 +9,6 @@
 import dask
 import dask.config
 
-from ..core import Status
 from .spec import ProcessInterface, SpecCluster
 
 logger = logging.getLogger(__name__)
@@ -89,6 +88,7 @@ def __init__(
         self.kwargs = kwargs
         self.name = name
         self.remote_python = remote_python
+        self.nprocs = self.kwargs.pop("nprocs", 1)
 
     async def start(self):
         import asyncssh  # import now to avoid adding to module startup time
@@ -125,11 +125,14 @@ async def start(self):
                 "'%s'"
                 % dumps(
                     {
-                        "cls": self.worker_class,
-                        "opts": {
-                            **self.kwargs,
-                            "name": self.name,
-                        },
+                        i: {
+                            "cls": self.worker_class,
+                            "opts": {
+                                **self.kwargs,
+                                "name": self.name,
+                            },
+                        }
+                        for i in range(self.nprocs)
                     }
                 ),
             ]
@@ -138,15 +141,14 @@ async def start(self):
         self.proc = await self.connection.create_process(cmd)
 
         # We watch stderr in order to get the address, then we return
-        while True:
+        started_workers = 0
+        while started_workers < self.nprocs:
             line = await self.proc.stderr.readline()
             if not line.strip():
                 raise Exception("Worker failed to start")
             logger.info(line.strip())
             if "worker at" in line:
-                self.address = line.split("worker at:")[1].strip()
-                self.status = Status.running
-                break
+                started_workers += 1
         logger.debug("%s", line)
         await super().start()
 
@@ -316,6 +318,17 @@ def SSHCluster(
     ... )
     >>> client = Client(cluster)
 
+    Create a cluster with two workers on each host:
+
+    >>> from dask.distributed import Client, SSHCluster
+    >>> cluster = SSHCluster(
+    ...     ["localhost", "localhost", "localhost", "localhost"],
+    ...     connect_options={"known_hosts": None},
+    ...     worker_options={"nthreads": 2, "nprocs": 2},
+    ...     scheduler_options={"port": 0, "dashboard_address": ":8797"}
+    ... )
+    >>> client = Client(cluster)
+
     An example using a different worker class, in particular the
     ``CUDAWorker`` from the ``dask-cuda`` project:
 


Test patch:
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -44,6 +44,25 @@ async def test_basic():
         assert "SSH" in repr(cluster)
 
 
+@pytest.mark.asyncio
+async def test_nprocs():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_options=dict(known_hosts=None),
+        asynchronous=True,
+        scheduler_options={"idle_timeout": "5s"},
+        worker_options={"death_timeout": "5s", "nprocs": 2},
+    ) as cluster:
+        assert len(cluster.workers) == 2
+        async with Client(cluster, asynchronous=True) as client:
+            client.wait_for_workers(4)
+            result = await client.submit(lambda x: x + 1, 10)
+            assert result == 11
+        assert not cluster._supports_scaling
+
+        assert "SSH" in repr(cluster)
+
+
 @pytest.mark.asyncio
 async def test_keywords():
     async with SSHCluster(