Repository: modin-project/modin

Issue:
Modin stuck at to_csv operation 
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: macOS Big sur, also tested on WSL with windows 10
- **Modin version** (`modin.__version__`): 0.10.2
- **Python version**: Python 3.8.11
- **Code we can use to reproduce**:

### Describe the problem
I have a post-processing function to arrange forecast results from gluonts to a dataframe.
The function below works well and computes a dataframe. The function output is then saved as a csv file where raylet is getting stuck. (Not even defaulting to pandas)

### Source code / logs
I first initialize ray with a certain number of CPU count. I'm testing Modin with 1 CPU (8 cores)
```
import os
os.environ["MODIN_ENGINE"] = "ray"  # Modin will use Ray
import ray
ray.init(num_cpus=6)
```

Here the post-processing function
```
def transform_outputs(
    kpi_name: str,
    prediction_list: List[gluonts.model.forecast.SampleForecast],
    confidence_intervals: List[float] = [0.5, 0.9]
):
    
    list_dfs = []
    for forecast in tqdm(prediction_list):
        ns, horz = forecast.samples.shape
        dates = pd.date_range(forecast.start_date, freq=forecast.freq, periods=horz)
        forecast_serie = {}
        forecast_serie[f"{kpi_name}_predict"] = forecast.quantile(0.5)
        df_forecast = pd.DataFrame(forecast_serie)
        
        intervals_dfs = []
        for interval in confidence_intervals:
            lower_p = (1.0 - interval) / 2
            upper_p = (1.0 + interval) / 2

            percentage = str(round(interval*100))
            series={}
            series[f"{kpi_name}_lower_predict_{percentage}%"] = forecast.quantile(q=lower_p)
            series[f"{kpi_name}_upper_predict_{percentage}%"] = forecast.quantile(q=upper_p)
            intervals_dfs.append(pd.DataFrame(series))

        df_intervals = pd.concat(intervals_dfs, axis=1)
        df = pd.concat([df_forecast, df_intervals], axis=1)
        df.index = dates
        df['info'] = forecast.item_id
        list_dfs.append(df)

    return pd.concat(list_dfs, axis=0)

df_forecast_group = transform_outputs(ts_name=ts_name, prediction_list=list_forecasts, confidence_intervals=[0.5, 0.8, 0.9])
logger.info('finnished post processing and start saving')
df_forecast_group.to_csv(f"{output_filepath}/{file_name.replace('.csv', '_forecast.csv')}",
                             index=True,
                             date_format='%Y-%m-%d'
                             )
```






Discussion:
The .to_csv() method works when I force the dataframe to default to pandas by using the private method `DataFrame._to_pandas()`
Hello @mbrhd,

Can you provide a dataframe on which the `to_csv` operation stuck so that I can reproduce the problem for myself?

Resolving patch:
diff --git a/modin/core/execution/ray/common/utils.py b/modin/core/execution/ray/common/utils.py
--- a/modin/core/execution/ray/common/utils.py
+++ b/modin/core/execution/ray/common/utils.py
@@ -17,6 +17,9 @@
 import sys
 import psutil
 import warnings
+import asyncio
+
+import ray
 
 from modin.config import (
     StorageFormat,
@@ -107,8 +110,6 @@ def initialize_ray(
         What password to use when connecting to Redis.
         If not specified, ``modin.config.RayRedisPassword`` is used.
     """
-    import ray
-
     if not ray.is_initialized() or override_is_cluster:
         cluster = override_is_cluster or IsRayCluster.get()
         redis_address = override_redis_address or RayRedisAddress.get()
@@ -196,3 +197,54 @@ def initialize_ray(
         NPartitions._put(num_gpus)
     else:
         NPartitions._put(num_cpus)
+
+
+@ray.remote
+class SignalActor:  # pragma: no cover
+    """
+    Help synchronize across tasks and actors on cluster.
+
+    For details see: https://docs.ray.io/en/latest/advanced.html?highlight=signalactor#multi-node-synchronization-using-an-actor
+
+    Parameters
+    ----------
+    event_count : int
+        Number of events required for synchronization.
+    """
+
+    def __init__(self, event_count: int):
+        self.events = [asyncio.Event() for _ in range(event_count)]
+
+    def send(self, event_idx: int):
+        """
+        Indicate that event with `event_idx` has occured.
+
+        Parameters
+        ----------
+        event_idx : int
+        """
+        self.events[event_idx].set()
+
+    async def wait(self, event_idx: int):
+        """
+        Wait until event with `event_idx` has occured.
+
+        Parameters
+        ----------
+        event_idx : int
+        """
+        await self.events[event_idx].wait()
+
+    def is_set(self, event_idx: int) -> bool:
+        """
+        Check that event with `event_idx` had occured or not.
+
+        Parameters
+        ----------
+        event_idx : int
+
+        Returns
+        -------
+        bool
+        """
+        return self.events[event_idx].is_set()
diff --git a/modin/core/execution/ray/generic/io/io.py b/modin/core/execution/ray/generic/io/io.py
--- a/modin/core/execution/ray/generic/io/io.py
+++ b/modin/core/execution/ray/generic/io/io.py
@@ -18,8 +18,8 @@
 import pandas
 
 from modin.core.io import BaseIO
-from ray.util.queue import Queue
-from ray import get
+from modin.core.execution.ray.common.utils import SignalActor
+import ray
 
 
 class RayIO(BaseIO):
@@ -118,9 +118,7 @@ def to_csv(cls, qc, **kwargs):
         if not cls._to_csv_check_support(kwargs):
             return BaseIO.to_csv(qc, **kwargs)
 
-        # The partition id will be added to the queue, for which the moment
-        # of writing to the file has come
-        queue = Queue(maxsize=1)
+        signals = SignalActor.remote(len(qc._modin_frame._partitions) + 1)
 
         def func(df, **kw):
             """
@@ -134,40 +132,35 @@ def func(df, **kw):
                 Arguments to pass to ``pandas.to_csv(**kw)`` plus an extra argument
                 `partition_idx` serving as chunk index to maintain rows order.
             """
-            if kw["partition_idx"] != 0:
+            partition_idx = kw["partition_idx"]
+            # the copy is made to not implicitly change the input parameters;
+            # to write to an intermediate buffer, we need to change `path_or_buf` in kwargs
+            csv_kwargs = kwargs.copy()
+            if partition_idx != 0:
                 # we need to create a new file only for first recording
                 # all the rest should be recorded in appending mode
-                if "w" in kwargs["mode"]:
-                    kwargs["mode"] = kwargs["mode"].replace("w", "a")
+                if "w" in csv_kwargs["mode"]:
+                    csv_kwargs["mode"] = csv_kwargs["mode"].replace("w", "a")
                 # It is enough to write the header for the first partition
-                kwargs["header"] = False
+                csv_kwargs["header"] = False
 
             # for parallelization purposes, each partition is written to an intermediate buffer
-            path_or_buf = kwargs["path_or_buf"]
-            is_binary = "b" in kwargs["mode"]
-            if is_binary:
-                kwargs["path_or_buf"] = io.BytesIO()
-            else:
-                kwargs["path_or_buf"] = io.StringIO()
-            df.to_csv(**kwargs)
-            content = kwargs["path_or_buf"].getvalue()
-            kwargs["path_or_buf"].close()
-
-            # each process waits for its turn to write to a file;
-            # in case of violation of the order of receiving messages from the queue,
-            # the message is placed back
-            while True:
-                get_value = queue.get(block=True)
-                if get_value == kw["partition_idx"]:
-                    break
-                queue.put(get_value)
+            path_or_buf = csv_kwargs["path_or_buf"]
+            is_binary = "b" in csv_kwargs["mode"]
+            csv_kwargs["path_or_buf"] = io.BytesIO() if is_binary else io.StringIO()
+            df.to_csv(**csv_kwargs)
+            content = csv_kwargs["path_or_buf"].getvalue()
+            csv_kwargs["path_or_buf"].close()
+
+            # each process waits for its turn to write to a file
+            ray.get(signals.wait.remote(partition_idx))
 
             # preparing to write data from the buffer to a file
             with pandas.io.common.get_handle(
                 path_or_buf,
                 # in case when using URL in implicit text mode
                 # pandas try to open `path_or_buf` in binary mode
-                kwargs["mode"] if is_binary else kwargs["mode"] + "t",
+                csv_kwargs["mode"] if is_binary else csv_kwargs["mode"] + "t",
                 encoding=kwargs["encoding"],
                 errors=kwargs["errors"],
                 compression=kwargs["compression"],
@@ -177,13 +170,12 @@ def func(df, **kw):
                 handles.handle.write(content)
 
             # signal that the next process can start writing to the file
-            queue.put(get_value + 1)
-
+            ray.get(signals.send.remote(partition_idx + 1))
             # used for synchronization purposes
             return pandas.DataFrame()
 
         # signaling that the partition with id==0 can be written to the file
-        queue.put(0)
+        ray.get(signals.send.remote(0))
         result = qc._modin_frame._partition_mgr_cls.map_axis_partitions(
             axis=1,
             partitions=qc._modin_frame._partitions,
@@ -193,9 +185,8 @@ def func(df, **kw):
             enumerate_partitions=True,
             max_retries=0,
         )
-
         # pending completion
-        get([partition.oid for partition in result.flatten()])
+        ray.get([partition.oid for partition in result.flatten()])
 
     @staticmethod
     def _to_parquet_check_support(kwargs):
@@ -268,4 +259,4 @@ def func(df, **kw):
             lengths=None,
             enumerate_partitions=True,
         )
-        get([partition.oid for partition in result.flatten()])
+        ray.get([part.oid for row in result for part in row])


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -59,6 +59,15 @@
     import modin.pandas as pd
 else:
     import modin.experimental.pandas as pd
+
+try:
+    import ray
+
+    EXCEPTIONS = (ray.exceptions.WorkerCrashedError,)
+except ImportError:
+    EXCEPTIONS = ()
+
+
 from modin.config import NPartitions
 
 NPartitions.put(4)
@@ -143,7 +152,20 @@ def eval_to_file(modin_obj, pandas_obj, fn, extension, **fn_kwargs):
     unique_filename_pandas = get_unique_filename(extension=extension)
 
     try:
-        getattr(modin_obj, fn)(unique_filename_modin, **fn_kwargs)
+        # parameter `max_retries=0` is set for `to_csv` function on Ray engine,
+        # in order to increase the stability of tests, we repeat the call of
+        # the entire function manually
+        last_exception = None
+        for _ in range(3):
+            try:
+                getattr(modin_obj, fn)(unique_filename_modin, **fn_kwargs)
+            except EXCEPTIONS as exc:
+                last_exception = exc
+                continue
+            break
+        else:
+            raise last_exception
+
         getattr(pandas_obj, fn)(unique_filename_pandas, **fn_kwargs)
 
         assert assert_files_eq(unique_filename_modin, unique_filename_pandas)