Repository: modin-project/modin

Issue:
PERF: Stop overriding Ray's 2 GiB object store size limit on mac
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: macOS Monterey 
- **Computer model**: MacBook Pro (16-inch, 2019)
- **Memory**: 16 GB 2667 MHz DDR4
- **Processor**: 2.3 GHz 8-Core Intel Core i9
- **Modin version** (`modin.__version__`): cc713c5c9055f717ac891cd1fccbd08987e169cd
- **Python version**: 3.10.4
- **Code we can use to reproduce**:

```python
import modin.pandas as pd
import numpy as np
from modin.config import BenchmarkMode
import time

BenchmarkMode.put(True)

df = pd.DataFrame(np.random.randint(0, 100, size=(int(2**20), 2**8)))
start = time.time()
df.apply(lambda col: col + 1)
end = time.time()
print(f'apply took: {end - start}')
```

### Describe the problem
Ray limits the object store size on mac to 2 GiB, but Modin overrides this limit. In the script above, pandas takes 3.6 sec and modin takes 25 sec for the apply. If I instead initialize a default ray cluster with `ray.init()`, I get some spilling to disk, but the apply takes only 5.2 sec in Modin.

Spilling to disk is also bad (e.g. worse than pandas here), but it seems to be much better than using the slow object store.



Discussion:

Resolving patch:
diff --git a/modin/core/execution/ray/common/utils.py b/modin/core/execution/ray/common/utils.py
--- a/modin/core/execution/ray/common/utils.py
+++ b/modin/core/execution/ray/common/utils.py
@@ -17,6 +17,7 @@
 import sys
 import psutil
 from packaging import version
+from typing import Optional
 import warnings
 
 import ray
@@ -34,6 +35,8 @@
 )
 from modin.error_message import ErrorMessage
 
+_OBJECT_STORE_TO_SYSTEM_MEMORY_RATIO = 0.6
+
 ObjectIDType = ray.ObjectRef
 if version.parse(ray.__version__) >= version.parse("1.2.0"):
     from ray.util.client.common import ClientObjectRef
@@ -98,53 +101,7 @@ def initialize_ray(
     ray.init({', '.join([f'{k}={v}' for k,v in extra_init_kw.items()])})
 """,
             )
-            object_store_memory = Memory.get()
-            # In case anything failed above, we can still improve the memory for Modin.
-            if object_store_memory is None:
-                virtual_memory = psutil.virtual_memory().total
-                if sys.platform.startswith("linux"):
-                    shm_fd = os.open("/dev/shm", os.O_RDONLY)
-                    try:
-                        shm_stats = os.fstatvfs(shm_fd)
-                        system_memory = shm_stats.f_bsize * shm_stats.f_bavail
-                        if system_memory / (virtual_memory / 2) < 0.99:
-                            warnings.warn(
-                                f"The size of /dev/shm is too small ({system_memory} bytes). The required size "
-                                + f"at least half of RAM ({virtual_memory // 2} bytes). Please, delete files in /dev/shm or "
-                                + "increase size of /dev/shm with --shm-size in Docker. Also, you can set "
-                                + "the required memory size for each Ray worker in bytes to MODIN_MEMORY environment variable."
-                            )
-                    finally:
-                        os.close(shm_fd)
-                else:
-                    system_memory = virtual_memory
-                object_store_memory = int(0.6 * system_memory // 1e9 * 1e9)
-                # If the memory pool is smaller than 2GB, just use the default in ray.
-                if object_store_memory == 0:
-                    object_store_memory = None
-            else:
-                object_store_memory = int(object_store_memory)
-
-            mac_size_limit = getattr(
-                ray.ray_constants, "MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT", None
-            )
-            if (
-                sys.platform == "darwin"
-                and mac_size_limit is not None
-                and object_store_memory > mac_size_limit
-            ):
-                warnings.warn(
-                    "On Macs, Ray's performance is known to degrade with "
-                    + "object store size greater than "
-                    + f"{mac_size_limit / 2 ** 30:.4} GiB. Ray by default does "
-                    + "not allow setting an object store size greater than "
-                    + "that. Modin is overriding that default limit because "
-                    + "it would rather have a larger, slower object store "
-                    + "than spill to disk more often. To override Modin's "
-                    + "behavior, you can initialize Ray yourself."
-                )
-                os.environ["RAY_ENABLE_MAC_LARGE_OBJECT_STORE"] = "1"
-
+            object_store_memory = _get_object_store_memory()
             ray_init_kwargs = {
                 "num_cpus": CpuCount.get(),
                 "num_gpus": GpuCount.get(),
@@ -186,6 +143,65 @@ def initialize_ray(
         NPartitions._put(num_cpus)
 
 
+def _get_object_store_memory() -> Optional[int]:
+    """
+    Get the object store memory we should start Ray with, in bytes.
+
+    - If the ``Memory`` config variable is set, return that.
+    - On Linux, take system memory from /dev/shm. On other systems use total
+      virtual memory.
+    - On Mac, never return more than Ray-specified upper limit.
+
+    Returns
+    -------
+    Optional[int]
+        The object store memory size in bytes, or None if we should use the Ray
+        default.
+    """
+    object_store_memory = Memory.get()
+    if object_store_memory is not None:
+        return object_store_memory
+    virtual_memory = psutil.virtual_memory().total
+    if sys.platform.startswith("linux"):
+        shm_fd = os.open("/dev/shm", os.O_RDONLY)
+        try:
+            shm_stats = os.fstatvfs(shm_fd)
+            system_memory = shm_stats.f_bsize * shm_stats.f_bavail
+            if system_memory / (virtual_memory / 2) < 0.99:
+                warnings.warn(
+                    f"The size of /dev/shm is too small ({system_memory} bytes). The required size "
+                    + f"at least half of RAM ({virtual_memory // 2} bytes). Please, delete files in /dev/shm or "
+                    + "increase size of /dev/shm with --shm-size in Docker. Also, you can can override the memory "
+                    + "size for each Ray worker (in bytes) to the MODIN_MEMORY environment variable."
+                )
+        finally:
+            os.close(shm_fd)
+    else:
+        system_memory = virtual_memory
+    bytes_per_gb = 1e9
+    object_store_memory = int(
+        _OBJECT_STORE_TO_SYSTEM_MEMORY_RATIO
+        * system_memory
+        // bytes_per_gb
+        * bytes_per_gb
+    )
+    if object_store_memory == 0:
+        return None
+    # Versions of ray with MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT don't allow us
+    # to initialize ray with object store size larger than that constant.
+    # For background see https://github.com/ray-project/ray/issues/20388
+    mac_size_limit = getattr(
+        ray.ray_constants, "MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT", None
+    )
+    if (
+        sys.platform == "darwin"
+        and mac_size_limit is not None
+        and object_store_memory > mac_size_limit
+    ):
+        object_store_memory = mac_size_limit
+    return object_store_memory
+
+
 def deserialize(obj):
     """
     Deserialize a Ray object.


Test patch: