Repository: modin-project/modin

Issue:
FEAT: HDK: Arrow-based columns concatenation of frames with trivial index
The outer join is not supported by HDK and, thus, frames concatenation with outer join always defaults to pandas. However, if all the frames have a trivial index, we can simply concatenate the columns with arrow. To improve the performance, arrow-based concatenation should be implemented.


Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
@@ -1290,7 +1290,15 @@ def _join_by_index(self, other_modin_frames, how, sort, ignore_index):
         HdkOnNativeDataframe
             The new frame.
         """
-        check_join_supported(how)
+        try:
+            check_join_supported(how)
+        except NotImplementedError as err:
+            # The outer join is not supported by HDK, however, if all the frames
+            # have a trivial index, we can simply concatenate the columns with arrow.
+            if (frame := self._join_arrow_columns(other_modin_frames)) is not None:
+                return frame
+            raise err
+
         lhs = self._maybe_materialize_rowid()
         reset_index_names = False
         new_columns_dtype = self.columns.dtype
@@ -1360,6 +1368,50 @@ def _join_by_index(self, other_modin_frames, how, sort, ignore_index):
 
         return lhs
 
+    def _join_arrow_columns(self, other_modin_frames):
+        """
+        Join arrow table columns.
+
+        If all the frames have a trivial index and an arrow
+        table in partitions, concatenate the table columns.
+
+        Parameters
+        ----------
+        other_modin_frames : list of HdkOnNativeDataframe
+            Frames to join with.
+
+        Returns
+        -------
+        HdkOnNativeDataframe or None
+        """
+        frames = [self] + other_modin_frames
+        if all(
+            f._index_cols is None
+            # Make sure all the frames have an arrow table in partitions. The
+            # method _execute() is no-op if the frame is already materialized
+            # and always returns None.
+            and (f._execute() or f._has_arrow_table())
+            for f in frames
+        ):
+            tables = [f._partitions[0][0].get() for f in frames]
+            column_names = [c for t in tables for c in t.column_names]
+            if len(column_names) != len(set(column_names)):
+                raise NotImplementedError("Duplicate column names")
+            max_len = max(len(t) for t in tables)
+            columns = [c for t in tables for c in t.columns]
+            # Make all columns of the same length, if required.
+            for i, col in enumerate(columns):
+                if len(col) < max_len:
+                    columns[i] = pyarrow.chunked_array(
+                        col.chunks + [pyarrow.nulls(max_len - len(col), col.type)]
+                    )
+            return self.from_arrow(
+                at=pyarrow.table(columns, column_names),
+                columns=[c for f in frames for c in f.columns],
+                encode_col_names=False,
+            )
+        return None
+
     def concat(
         self,
         axis: Union[int, Axis],
@@ -2269,6 +2321,12 @@ def _set_columns(self, new_columns):
         HdkOnNativeDataframe
             The new frame.
         """
+        if (
+            self.columns.identical(new_columns)
+            if isinstance(new_columns, Index)
+            else all(self.columns == new_columns)
+        ):
+            return self
         exprs = self._index_exprs()
         for old, new in zip(self.columns, new_columns):
             expr = self.ref(old)
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
@@ -224,9 +224,7 @@ def check_join_supported(join_type: str):
     None
     """
     if join_type not in ("inner", "left"):
-        raise NotImplementedError(
-            f"{join_type} join is not supported by the HDK engine"
-        )
+        raise NotImplementedError(f"{join_type} join")
 
 
 def check_cols_to_join(what, df, col_names):


Test patch:
diff --git a/modin/pandas/test/test_concat.py b/modin/pandas/test/test_concat.py
--- a/modin/pandas/test/test_concat.py
+++ b/modin/pandas/test/test_concat.py
@@ -84,8 +84,15 @@ def test_concat_on_index():
     )
 
 
-def test_concat_on_column():
+@pytest.mark.parametrize("no_dup_cols", [True, False])
+@pytest.mark.parametrize("different_len", [True, False])
+def test_concat_on_column(no_dup_cols, different_len):
     df, df2 = generate_dfs()
+    if no_dup_cols:
+        df = df.drop(set(df.columns) & set(df2.columns), axis="columns")
+    if different_len:
+        df = pandas.concat([df, df], ignore_index=True)
+
     modin_df, modin_df2 = from_pandas(df), from_pandas(df2)
 
     df_equals(