Repository: modin-project/modin

Issue:
Possible to remove to_pandas calls in merge and join operations
In query_compiler.merge and query_compiler.join we use `to_pandas()` eg [right_pandas = right.to_pandas()](https://github.com/modin-project/modin/blob/7ef544f4467ddea18cfbb51ad2a6fcbbb12c0db3/modin/core/storage_formats/pandas/query_compiler.py#L525) in This operation in the main thread is blocking. This could be expensive if the right dataframe size is too large. 

It is possible to remove the `to_pandas` calls by replacing `apply_full_axis` with broadcast_apply_full_axis and pass the `right` dataframe to `broadcast_apply_full_axis`  as _Originally suggested by @YarShev in https://github.com/modin-project/modin/issues/5524#issuecomment-1880982441_ and 





Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -522,8 +522,6 @@ def merge(self, right, **kwargs):
         sort = kwargs.get("sort", False)
 
         if how in ["left", "inner"] and left_index is False and right_index is False:
-            right_pandas = right.to_pandas()
-
             kwargs["sort"] = False
 
             def should_keep_index(left, right):
@@ -542,7 +540,7 @@ def should_keep_index(left, right):
                 return keep_index
 
             def map_func(
-                left, *axis_lengths, right=right_pandas, kwargs=kwargs, **service_kwargs
+                left, right, *axis_lengths, kwargs=kwargs, **service_kwargs
             ):  # pragma: no cover
                 df = pandas.merge(left, right, **kwargs)
 
@@ -571,7 +569,7 @@ def map_func(
             if self._modin_frame.has_materialized_columns:
                 if left_on is None and right_on is None:
                     if on is None:
-                        on = [c for c in self.columns if c in right_pandas.columns]
+                        on = [c for c in self.columns if c in right.columns]
                     _left_on, _right_on = on, on
                 else:
                     if left_on is None or right_on is None:
@@ -583,7 +581,7 @@ def map_func(
                 try:
                     new_columns, left_renamer, right_renamer = join_columns(
                         self.columns,
-                        right_pandas.columns,
+                        right.columns,
                         _left_on,
                         _right_on,
                         kwargs.get("suffixes", ("_x", "_y")),
@@ -595,15 +593,15 @@ def map_func(
                 else:
                     # renamers may contain columns from 'index', so trying to merge index and column dtypes here
                     right_index_dtypes = (
-                        right_pandas.index.dtypes
-                        if isinstance(right_pandas.index, pandas.MultiIndex)
+                        right.index.dtypes
+                        if isinstance(right.index, pandas.MultiIndex)
                         else pandas.Series(
-                            [right_pandas.index.dtype], index=[right_pandas.index.name]
+                            [right.index.dtype], index=[right.index.name]
                         )
                     )
-                    right_dtypes = pandas.concat(
-                        [right_pandas.dtypes, right_index_dtypes]
-                    )[right_renamer.keys()].rename(right_renamer)
+                    right_dtypes = pandas.concat([right.dtypes, right_index_dtypes])[
+                        right_renamer.keys()
+                    ].rename(right_renamer)
 
                     left_index_dtypes = (
                         self._modin_frame._index_cache.maybe_get_dtypes()
@@ -618,10 +616,11 @@ def map_func(
                     new_dtypes = ModinDtypes.concat([left_dtypes, right_dtypes])
 
             new_self = self.__constructor__(
-                self._modin_frame.apply_full_axis(
+                self._modin_frame.broadcast_apply_full_axis(
                     axis=1,
                     func=map_func,
                     enumerate_partitions=how == "left",
+                    other=right._modin_frame,
                     # We're going to explicitly change the shape across the 1-axis,
                     # so we want for partitioning to adapt as well
                     keep_partitioning=False,
@@ -629,8 +628,8 @@ def map_func(
                         self._modin_frame, right._modin_frame, axis=1
                     ),
                     new_columns=new_columns,
-                    dtypes=new_dtypes,
                     sync_labels=False,
+                    dtypes=new_dtypes,
                     pass_axis_lengths_to_partitions=how == "left",
                 )
             )
@@ -641,20 +640,19 @@ def map_func(
             # materialized quite often compared to the indexes.
             keep_index = False
             if self._modin_frame.has_materialized_index:
-                keep_index = should_keep_index(self, right_pandas)
+                keep_index = should_keep_index(self, right)
             else:
                 # Have to trigger columns materialization. Hope they're already available at this point.
                 if left_on is not None and right_on is not None:
                     keep_index = any(
-                        o not in right_pandas.columns
+                        o not in right.columns
                         and o in left_on
                         and o not in self.columns
                         for o in right_on
                     )
                 elif on is not None:
                     keep_index = any(
-                        o not in right_pandas.columns and o not in self.columns
-                        for o in on
+                        o not in right.columns and o not in self.columns for o in on
                     )
 
             if sort:
@@ -685,13 +683,12 @@ def join(self, right, **kwargs):
         sort = kwargs.get("sort", False)
 
         if how in ["left", "inner"]:
-            right_pandas = right.to_pandas()
 
-            def map_func(left, right=right_pandas, kwargs=kwargs):  # pragma: no cover
+            def map_func(left, right, kwargs=kwargs):  # pragma: no cover
                 return pandas.DataFrame.join(left, right, **kwargs)
 
             new_self = self.__constructor__(
-                self._modin_frame.apply_full_axis(
+                self._modin_frame.broadcast_apply_full_axis(
                     axis=1,
                     func=map_func,
                     # We're going to explicitly change the shape across the 1-axis,
@@ -700,6 +697,7 @@ def map_func(left, right=right_pandas, kwargs=kwargs):  # pragma: no cover
                     num_splits=merge_partitioning(
                         self._modin_frame, right._modin_frame, axis=1
                     ),
+                    other=right._modin_frame,
                 )
             )
             return new_self.sort_rows_by_column_values(on) if sort else new_self


Test patch: