Repository: modin-project/modin

Issue:
Do not set Ray's runtime env in case of a single-node execution
Specifying a non-empty runtime env at `ray.init()` causes a decent overhead on starting up the workers:
https://github.com/modin-project/modin/blob/c2b8d8e20b37a6ea6d3111112181984e51f3208b/modin/core/execution/ray/common/utils.py#L89
We can get rid of this in a single-node case by simply setting the required env variables in the main process. See this for more info:
https://github.com/modin-project/modin/issues/5157#issuecomment-1500225150


Discussion:

Resolving patch:
diff --git a/modin/core/execution/ray/common/utils.py b/modin/core/execution/ray/common/utils.py
--- a/modin/core/execution/ray/common/utils.py
+++ b/modin/core/execution/ray/common/utils.py
@@ -77,6 +77,10 @@ def initialize_ray(
         What password to use when connecting to Redis.
         If not specified, ``modin.config.RayRedisPassword`` is used.
     """
+    # We need these vars to be set for each Ray's worker in order to ensure that
+    # the `pandas` module has been fully imported inside of each process before
+    # any execution begins:
+    # https://github.com/modin-project/modin/pull/4603
     env_vars = {"__MODIN_AUTOIMPORT_PANDAS__": "1"}
     if GithubCI.get():
         # need these to write parquet to the moto service mocking s3.
@@ -86,14 +90,14 @@ def initialize_ray(
                 "AWS_SECRET_ACCESS_KEY": CIAWSSecretAccessKey.get(),
             }
         )
-    extra_init_kw = {"runtime_env": {"env_vars": env_vars}}
+    extra_init_kw = {}
+    is_cluster = override_is_cluster or IsRayCluster.get()
     if not ray.is_initialized() or override_is_cluster:
-        cluster = override_is_cluster or IsRayCluster.get()
         redis_address = override_redis_address or RayRedisAddress.get()
         redis_password = (
             (
                 ray.ray_constants.REDIS_DEFAULT_PASSWORD
-                if cluster
+                if is_cluster
                 else RayRedisPassword.get()
             )
             if override_redis_password is None
@@ -101,7 +105,8 @@ def initialize_ray(
             else override_redis_password or RayRedisPassword.get()
         )
 
-        if cluster:
+        if is_cluster:
+            extra_init_kw["runtime_env"] = {"env_vars": env_vars}
             # We only start ray in a cluster setting for the head node.
             ray.init(
                 address=redis_address or "auto",
@@ -131,6 +136,14 @@ def initialize_ray(
                 "_memory": object_store_memory,
                 **extra_init_kw,
             }
+            # It should be enough to simply set the required variables for the main process
+            # for Ray to automatically propagate them to each new worker on the same node.
+            # Although Ray doesn't guarantee this behavior it works as expected most of the
+            # time and doesn't enforce us with any overhead that Ray's native `runtime_env`
+            # is usually causing. You can visit this gh-issue for more info:
+            # https://github.com/modin-project/modin/issues/5157#issuecomment-1500225150
+            for key, value in env_vars.items():
+                os.environ[key] = value
             ray.init(**ray_init_kwargs)
 
         if StorageFormat.get() == "Cudf":
@@ -146,13 +159,19 @@ def initialize_ray(
 
     # Now ray is initialized, check runtime env config - especially useful if we join
     # an externally pre-configured cluster
-    env_vars = ray.get_runtime_context().runtime_env.get("env_vars", {})
-    for varname, varvalue in extra_init_kw["runtime_env"]["env_vars"].items():
-        if str(env_vars.get(varname, "")) != str(varvalue):
-            ErrorMessage.single_warning(
-                "When using a pre-initialized Ray cluster, please ensure that the runtime env "
-                + f"sets environment variable {varname} to {varvalue}"
-            )
+    runtime_env_vars = ray.get_runtime_context().runtime_env.get("env_vars", {})
+    for varname, varvalue in env_vars.items():
+        if str(runtime_env_vars.get(varname, "")) != str(varvalue):
+            if is_cluster or (
+                # Here we relax our requirements for a non-cluster case allowing for the `env_vars`
+                # to be set at least as a process environment variable
+                not is_cluster
+                and os.environ.get(varname, "") != str(varvalue)
+            ):
+                ErrorMessage.single_warning(
+                    "When using a pre-initialized Ray cluster, please ensure that the runtime env "
+                    + f"sets environment variable {varname} to {varvalue}"
+                )
 
     num_cpus = int(ray.cluster_resources()["CPU"])
     num_gpus = int(ray.cluster_resources().get("GPU", 0))


Test patch: