Repository: modin-project/modin

Issue:
Add a mechanism that allows to install additional packages in a remote environment.



Discussion:
Just started.

Resolving patch:
diff --git a/modin/experimental/cloud/cluster.py b/modin/experimental/cloud/cluster.py
--- a/modin/experimental/cloud/cluster.py
+++ b/modin/experimental/cloud/cluster.py
@@ -110,6 +110,7 @@ def __init__(
         worker_count: int = 4,
         head_node_type: str = None,
         worker_node_type: str = None,
+        add_conda_packages: list = None,
     ):
         """
         Prepare the cluster manager. It needs to know a few things:
@@ -118,6 +119,7 @@ def __init__(
             * cluster name
             * worker count
             * head and worker node instance types (can be omitted to default to provider-defined)
+            * custom conda packages for remote environment
         """
 
         self.provider = provider
@@ -126,6 +128,7 @@ def __init__(
         self.worker_count = worker_count
         self.head_node_type = head_node_type or provider.default_head_type
         self.worker_node_type = worker_node_type or provider.default_worker_type
+        self.add_conda_packages = add_conda_packages
 
         self.old_backends = None
         self.connection: Connection = None
@@ -211,6 +214,7 @@ def create(
     workers: int = 4,
     head_node: str = None,
     worker_node: str = None,
+    add_conda_packages: list = None,
     cluster_type: str = "rayscale",
 ) -> BaseCluster:
     """
@@ -246,6 +250,8 @@ def create(
         What machine type to use for head node in the cluster.
     worker_node : str, optional
         What machine type to use for worker nodes in the cluster.
+    add_conda_packages : list, optional
+        Custom conda packages for remote environments.
     cluster_type : str, optional
         How to spawn the cluster.
         Currently spawning by Ray autoscaler ("rayscale" for general and "omnisci" for Omnisci-based) is supported
@@ -294,6 +300,7 @@ def create(
         worker_count=workers,
         head_node_type=head_node,
         worker_node_type=worker_node,
+        add_conda_packages=add_conda_packages,
     )
     instance.spawn(wait=False)
     return instance
diff --git a/modin/experimental/cloud/local_cluster.py b/modin/experimental/cloud/local_cluster.py
--- a/modin/experimental/cloud/local_cluster.py
+++ b/modin/experimental/cloud/local_cluster.py
@@ -83,6 +83,7 @@ def __init__(
         worker_count=_UNUSED,
         head_node_type=_UNUSED,
         worker_node_type=_UNUSED,
+        add_conda_packages=_UNUSED,
     ):
         assert (
             provider == "local"
@@ -95,6 +96,7 @@ def __init__(
                 worker_count,
                 head_node_type,
                 worker_node_type,
+                add_conda_packages,
             )
         ):
             warnings.warn(
diff --git a/modin/experimental/cloud/rayscale.py b/modin/experimental/cloud/rayscale.py
--- a/modin/experimental/cloud/rayscale.py
+++ b/modin/experimental/cloud/rayscale.py
@@ -141,18 +141,25 @@ def __make_config(self):
 
         return _bootstrap_config(config)
 
-    @classmethod
-    def _conda_requirements(cls):
+    def _conda_requirements(self):
+        import shlex
+
         reqs = []
 
-        reqs.append(f"python=={cls._get_python_version()}")
+        reqs.append(f"python=={self._get_python_version()}")
+
+        if self.add_conda_packages:
+            reqs.extend(self.add_conda_packages)
+
+        # this is needed, for example, for dependencies that
+        # looks like: "scikit-learn>=0.23"
+        reqs_with_quotes = [shlex.quote(req) for req in reqs]
 
-        return reqs
+        return reqs_with_quotes
 
-    @classmethod
-    def _update_conda_requirements(cls, setup_commands: str):
+    def _update_conda_requirements(self, setup_commands: str):
         return setup_commands.replace(
-            "{{CONDA_PACKAGES}}", " ".join(cls._conda_requirements())
+            "{{CONDA_PACKAGES}}", " ".join(self._conda_requirements())
         )
 
     @staticmethod


Test patch:
diff --git a/modin/experimental/cloud/test/test_cloud.py b/modin/experimental/cloud/test/test_cloud.py
--- a/modin/experimental/cloud/test/test_cloud.py
+++ b/modin/experimental/cloud/test/test_cloud.py
@@ -12,8 +12,10 @@
 # governing permissions and limitations under the License.
 
 import sys
+import unittest.mock as mock
 import pytest
 from modin.experimental.cloud.rayscale import RayCluster
+from modin.experimental.cloud.cluster import Provider
 
 
 @pytest.mark.parametrize(
@@ -33,7 +35,17 @@ def test_update_conda_requirements(setup_commands_source):
     micro = sys.version_info.micro
     python_version = f"python=={major}.{minor}.{micro}"
 
-    setup_commands_result = RayCluster._update_conda_requirements(setup_commands_source)
+    with mock.patch(
+        "modin.experimental.cloud.rayscale._bootstrap_config", lambda config: config
+    ):
+        ray_cluster = RayCluster(
+            Provider(name="aws"), add_conda_packages=["scikit-learn>=0.23"]
+        )
+
+    setup_commands_result = ray_cluster._update_conda_requirements(
+        setup_commands_source
+    )
 
     assert python_version in setup_commands_result
+    assert "scikit-learn>=0.23" in setup_commands_result
     assert "{{CONDA_PACKAGES}}" not in setup_commands_result