Repository: modin-project/modin

Issue:
Column rename operation fails when converting from Modin DataFrame to Ray Dataset
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: MacOS 12.3.1
- **Modin version** (`modin.__version__`): 0.15.2
- **Python version**: 3.8.13
- **Code we can use to reproduce**: https://gist.github.com/geoffreyangus/5e9fdfd711d9776e76053840fb332274

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem

I am attempting to do the following:

1. Convert Modin Series into a Modin DataFrame, while changing the column name using the `to_frame(name="value")`
2. Convert the Modin DataFrame to a Ray Dataset
3. run `map_batches` using the new column name (in this example, "value")
4. Convert the Ray Dataset back to a Modin DataFrame

I am getting an error in step (3) because "value" cannot be found in the Pandas DataFrame objects operated upon in the `map_batches` call.

### Expected Behavior

I expect column name assignment when calling `to_frame(name=<NAME>)`.

Interestingly enough, if the DataFrame is materialized beforehand (using `DataFrame._to_pandas()`), the column name is correctly renamed as a side effect.

### Source code / logs

A minimal repro can be found in the Gist above.



Discussion:
@geoffreyangus thank you for reporting this bug. I can reproduce it at Modin da7bd1a14752895a38342b2faa3220a8db4cb7ff. 

The bug is that `from_modin` [calls](https://github.com/ray-project/ray/blob/77e2ef2eb630cb7caa7ca31b3c406b45f8643192/python/ray/data/read_api.py#L835) Modin's `unwrap_partitions`, and Modin's `unwrap_partitions` doesn't apply deferred metadata updates to the modin frame before getting the data. I'll fix that.

Resolving patch:
diff --git a/modin/distributed/dataframe/pandas/partitions.py b/modin/distributed/dataframe/pandas/partitions.py
--- a/modin/distributed/dataframe/pandas/partitions.py
+++ b/modin/distributed/dataframe/pandas/partitions.py
@@ -50,18 +50,21 @@ def unwrap_partitions(api_layer_object, axis=None, get_ip=False):
             f"Only API Layer objects may be passed in here, got {type(api_layer_object)} instead."
         )
 
+    modin_frame = api_layer_object._query_compiler._modin_frame
+    modin_frame._propagate_index_objs(None)
     if axis is None:
 
         def _unwrap_partitions():
+            [p.drain_call_queue() for p in modin_frame._partitions.flatten()]
             if get_ip:
                 return [
                     [(partition._ip_cache, partition._data) for partition in row]
-                    for row in api_layer_object._query_compiler._modin_frame._partitions
+                    for row in modin_frame._partitions
                 ]
             else:
                 return [
                     [partition._data for partition in row]
-                    for row in api_layer_object._query_compiler._modin_frame._partitions
+                    for row in modin_frame._partitions
                 ]
 
         actual_engine = type(
@@ -76,8 +79,8 @@ def _unwrap_partitions():
             f"Do not know how to unwrap '{actual_engine}' underlying partitions"
         )
     else:
-        partitions = api_layer_object._query_compiler._modin_frame._partition_mgr_cls.axis_partition(
-            api_layer_object._query_compiler._modin_frame._partitions, axis ^ 1
+        partitions = modin_frame._partition_mgr_cls.axis_partition(
+            modin_frame._partitions, axis ^ 1
         )
         return [
             part.force_materialization(get_ip=get_ip).unwrap(


Test patch:
diff --git a/modin/test/test_partition_api.py b/modin/test/test_partition_api.py
--- a/modin/test/test_partition_api.py
+++ b/modin/test/test_partition_api.py
@@ -54,27 +54,39 @@
 
 
 @pytest.mark.parametrize("axis", [None, 0, 1])
-def test_unwrap_partitions(axis):
+@pytest.mark.parametrize("reverse_index", [True, False])
+@pytest.mark.parametrize("reverse_columns", [True, False])
+def test_unwrap_partitions(axis, reverse_index, reverse_columns):
     data = test_data["int_data"]
-    df = pd.DataFrame(data)
 
+    def get_df(lib, data):
+        df = lib.DataFrame(data)
+        if reverse_index:
+            df.index = df.index[::-1]
+        if reverse_columns:
+            df.columns = df.columns[::-1]
+        return df
+
+    df = get_df(pd, data)
+    # `df` should not have propagated the index and column updates to its
+    # partitions yet. The partitions of `expected_df` should have the updated
+    # metadata because we construct `expected_df` directly from the updated
+    # pandas dataframe.
+    expected_df = pd.DataFrame(get_df(pandas, data))
+    expected_partitions = expected_df._query_compiler._modin_frame._partitions
     if axis is None:
-        expected_partitions = df._query_compiler._modin_frame._partitions
         actual_partitions = np.array(unwrap_partitions(df, axis=axis))
-        assert (
-            expected_partitions.shape[0] == actual_partitions.shape[0]
-            and expected_partitions.shape[1] == expected_partitions.shape[1]
-        )
+        assert expected_partitions.shape == actual_partitions.shape
         for row_idx in range(expected_partitions.shape[0]):
             for col_idx in range(expected_partitions.shape[1]):
-                assert (
-                    expected_partitions[row_idx][col_idx]._data
-                    == actual_partitions[row_idx][col_idx]
+                df_equals(
+                    get_func(expected_partitions[row_idx][col_idx]._data),
+                    get_func(actual_partitions[row_idx][col_idx]),
                 )
     else:
         expected_axis_partitions = (
-            df._query_compiler._modin_frame._partition_mgr_cls.axis_partition(
-                df._query_compiler._modin_frame._partitions, axis ^ 1
+            expected_df._query_compiler._modin_frame._partition_mgr_cls.axis_partition(
+                expected_partitions, axis ^ 1
             )
         )
         expected_axis_partitions = [