Repository: modin-project/modin

Issue:
BUG: HDK: Unable to concatenate tables with different number of non-numeric columns
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
import modin.pandas as pd

df1 = pd.DataFrame({"A": [1, 2], "B": ["1", "2"]})
df2 = pd.DataFrame({"A": [1, 2]})
df = pd.concat([df1, df2])
print(df)
```


### Issue Description

Unable to concatenate tables with different number of non-numeric columns. If the columns are numeric, concatenation works, but it's ~100 times slower, than pandas (see https://github.com/intel-ai/hdk/issues/182)

### Expected Behavior

```
   A     B
0  1     1
1  2     2
0  1  None
1  2  None
```

### Error Logs

<details>

```python-traceback

modin/experimental/core/execution/native/implementations/hdk_on_native/hdk_worker.py", line 90, in executeRA
    ra = cls._calcite.process(query, db_name="hdk")
  File "_sql.pyx", line 41, in pyhdk._sql.Calcite.process
ValueError: Cannot compute compatible row type for arguments to set op: RecordType(BIGINT __index__, BIGINT A, VARCHAR(50) B), RecordType(BIGINT __index__, BIGINT A, BIGINT B)

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
@@ -30,6 +30,7 @@
 from modin.pandas.indexing import is_range_like
 from modin.utils import MODIN_UNNAMED_SERIES_LABEL
 import pandas as pd
+import typing
 from typing import List, Hashable, Optional, Tuple, Union
 
 from ..df_algebra import (
@@ -969,6 +970,19 @@ def _union_all(
         HdkOnNativeDataframe
             The new frame.
         """
+        # In case of different number of columns, HDK performs
+        # slowly and supports only numeric column types.
+        # See https://github.com/intel-ai/hdk/issues/182
+        # To work around this issue, perform concatenation
+        # with arrow.
+        if (
+            len(other_modin_frames) == 0
+            or len(self.columns) == 0
+            or any((len(f.columns) != len(self.columns) for f in other_modin_frames))
+            or any((set(f._dtypes) != set(self._dtypes) for f in other_modin_frames))
+        ):
+            return self._union_all_arrow(other_modin_frames, join, sort, ignore_index)
+
         # determine output columns
         new_cols_map = OrderedDict()
         for col in self.columns:
@@ -1016,8 +1030,8 @@ def _union_all(
                         exprs[col] = frame.ref(col)
                 else:
                     assert index_width == 1, "unexpected index width"
-                    aligned_index = ["__index__"]
-                    exprs["__index__"] = frame.ref("__rowid__")
+                    aligned_index = self._mangle_index_names([None])
+                    exprs[aligned_index[0]] = frame.ref("__rowid__")
                     aligned_index_dtypes = [get_dtype(int)]
                     uses_rowid = True
                 aligned_dtypes = aligned_index_dtypes + new_dtypes
@@ -1054,6 +1068,202 @@ def _union_all(
 
         return new_frame
 
+    def _union_all_arrow(self, other_modin_frames, join, sort, ignore_index):
+        """
+        Concatenate frames' rows, using the PyArrow API.
+
+        Parameters
+        ----------
+        other_modin_frames : list of HdkOnNativeDataframe
+            Frames to concat.
+        join : {"outer", "inner"}
+            How to handle columns with mismatched names.
+            "inner" - drop such columns. "outer" - fill
+            with NULLs.
+        sort : bool
+            Sort unaligned columns for 'outer' join.
+        ignore_index : bool
+            Ignore index columns.
+
+        Returns
+        -------
+        HdkOnNativeDataframe
+            The new frame.
+        """
+
+        class FrameData:
+            def __init__(self, frame):
+                if not frame._has_arrow_table():
+                    frame._execute()
+                if not frame._has_arrow_table():
+                    raise NotImplementedError("PyArrow tables concatenation")
+                self.index = frame.index
+                self.columns = frame.columns
+                self.index_cols = frame._index_cols
+                self.table: pyarrow.Table = frame._partitions[0][0].arrow_table
+                # To be removed after #5567 merge
+                if self.index_cols is not None:
+                    self.index_cols = [f"F_{n}" for n in self.index_cols]
+
+        # This function to be removed after #5567 merge
+        def mangle_index_names(names):
+            return [
+                f"F___index__{i}_{'__None__' if n is None else n}"
+                for i, n in enumerate(names)
+            ]
+
+        frames: List[FrameData] = [FrameData(f) for f in [self] + other_modin_frames]
+        col_fields: typing.OrderedDict[Tuple[str, str], pyarrow.Field] = OrderedDict()
+
+        def add_col_field(table, col_name, table_col_name):
+            key = (col_name, table_col_name)
+            field = table.field(table_col_name)
+            cur_field = col_fields.get(key, None)
+            if (
+                (cur_field is None)
+                or pyarrow.types.is_null(cur_field.type)
+                or (
+                    (not pyarrow.types.is_string(cur_field.type))
+                    and (not pyarrow.types.is_null(field.type))
+                    and (
+                        pyarrow.types.is_string(field.type)
+                        or (field.type.bit_width > cur_field.type.bit_width)
+                    )
+                )
+            ):
+                col_fields[key] = field
+
+        if join == "outer":
+            frames = [f for f in frames if len(f.columns) != 0 or len(f.index) != 0]
+            for frame in frames:
+                table = frame.table
+                idx_width = 0 if frame.index_cols is None else len(frame.index_cols)
+                table_cols = table.column_names[idx_width:]
+                for col_name, table_col_name in zip(frame.columns, table_cols):
+                    add_col_field(table, col_name, table_col_name)
+        else:
+            col_names = {c for c in frames[0].columns} if len(frames) > 1 else []
+            if len(col_names) != 0:
+                for frame in frames[1:]:
+                    for name in list(col_names):
+                        if name not in frame.columns:
+                            col_names.remove(name)
+                if len(col_names) != 0:
+                    for frame in frames:
+                        table = frame.table
+                        idx_width = (
+                            0 if frame.index_cols is None else len(frame.index_cols)
+                        )
+                        table_cols = table.column_names[idx_width:]
+                        for col_name, table_col_name in zip(frame.columns, table_cols):
+                            if col_name in col_names:
+                                add_col_field(table, col_name, table_col_name)
+
+        if len(col_fields) == 0:
+            if ignore_index or len(frames) == 0:
+                idx = RangeIndex(0, sum(len(f.table) for f in frames))
+            else:
+                idx = frames[0].index.append([f.index for f in frames[1:]])
+            idx_cols = mangle_index_names(idx.names)
+            idx_df = pd.DataFrame(index=idx).reset_index()
+            union = pyarrow.Table.from_pandas(idx_df).rename_columns(idx_cols)
+        else:
+            # Process empty frames with non-empty index
+            for frame in frames:
+                if len(frame.table) == 0 and len(frame.index) != 0:
+                    if ignore_index:
+                        frame.index = pd.RangeIndex(0, len(frame.index))
+                    idx = frame.index
+                    frame.index_cols = mangle_index_names(idx.names)
+                    idx_df = pd.DataFrame(index=idx).reset_index()
+                    frame.table = pyarrow.Table.from_pandas(idx_df)
+                    frame.table = frame.table.rename_columns(frame.index_cols)
+
+            idx_cols = None
+            idx_table = None
+            idx_fields: typing.OrderedDict[str, pyarrow.Field] = OrderedDict()
+
+            if not ignore_index:
+                idx_cols = frames[0].index_cols
+                idx_equal = idx_cols is not None
+
+                if idx_equal:
+                    idx_width = len(idx_cols)
+                    idx_types = [frames[0].table.field(c).type for c in idx_cols]
+                    for frame in frames[1:]:
+                        table = frame.table
+                        frame_idx_cols = frame.index_cols
+                        if (
+                            (frame_idx_cols is None)
+                            or (len(frame_idx_cols) != idx_width)
+                            or any(
+                                idx_types[i] != table.field(frame_idx_cols[i]).type
+                                for i in range(idx_width)
+                            )
+                        ):
+                            idx_equal = False
+                            break
+
+                if idx_equal:
+                    idx = frames[0].index
+                    if isinstance(idx, MultiIndex):
+                        idx_cols = mangle_index_names(idx.names)
+                    else:
+                        idx_names = {f.index.name for f in frames}
+                        idx_names = [None] if len(idx_names) > 1 else [idx.name]
+                        idx_cols = mangle_index_names(idx_names)
+
+                    # Rename index columns
+                    for frame in frames:
+                        table = frame.table
+                        new_names = idx_cols + table.column_names[len(idx_cols) :]
+                        frame.table = table.rename_columns(new_names)
+
+                    for name in idx_cols:
+                        idx_fields[name] = frames[0].table.field(name)
+                else:
+                    # Align index columns
+                    idx = frames[0].index.append([f.index for f in frames[1:]])
+                    idx_cols = mangle_index_names(idx.names)
+                    idx_df = pd.DataFrame(index=idx).reset_index()
+                    idx_table = pyarrow.Table.from_pandas(idx_df)
+                    idx_table = idx_table.rename_columns(idx_cols)
+
+            if sort:
+                col_fields = OrderedDict(sorted(col_fields.items(), key=lambda i: i[0]))
+
+            schema = pyarrow.schema(
+                list(idx_fields.values()) + list(col_fields.values())
+            )
+
+            tables = []
+            for frame in frames:
+                data = []
+                table = frame.table
+                col_names = table.column_names
+                for field in schema:
+                    if field.name in col_names:
+                        data.append(table.column(field.name))
+                    else:
+                        data.append(pyarrow.nulls(len(table), field.type))
+                tables.append(pyarrow.table(data, schema=schema))
+
+            union = pyarrow.concat_tables(tables)
+
+            if idx_table is not None:
+                for i, name in enumerate(idx_table.column_names):
+                    union = union.add_column(i, idx_table.field(i), idx_table.column(i))
+
+        # To be removed after #5567 merge
+        if idx_cols is not None:
+            idx_cols = [n[2:] for n in idx_cols]
+
+        return self.from_arrow(
+            union,
+            index_cols=idx_cols,
+            columns=[k[0] for k in col_fields.keys()],
+        )
+
     def _join_by_index(self, other_modin_frames, how, sort, ignore_index):
         """
         Perform equi-join operation for multiple frames by index columns.
@@ -1174,14 +1384,14 @@ def concat(
             The new frame.
         """
         axis = Axis(axis)
-        if not other_modin_frames:
-            return self
-
         if axis == Axis.ROW_WISE:
             return self._union_all(
                 axis.value, other_modin_frames, join, sort, ignore_index
             )
 
+        if not other_modin_frames:
+            return self
+
         base = self
         for frame in other_modin_frames:
             base = base._find_common_projections_base(frame)
@@ -2505,7 +2715,7 @@ def from_arrow(cls, at, index_cols=None, index=None, columns=None):
 
         if columns is not None:
             new_columns = columns
-            new_index = pd.RangeIndex(at.num_rows) if index is None else index
+            new_index = index
         elif index_cols:
             data_cols = [col for col in at.column_names if col not in index_cols]
             new_columns = pd.Index(data=data_cols, dtype="O")
@@ -2513,8 +2723,10 @@ def from_arrow(cls, at, index_cols=None, index=None, columns=None):
         else:
             assert index is None
             new_columns = pd.Index(data=at.column_names, dtype="O")
-            new_index = pd.RangeIndex(at.num_rows)
+            new_index = None
 
+        dtype_index = [] if index_cols is None else list(index_cols)
+        dtype_index.extend(new_columns)
         new_dtypes = []
 
         for col in at.columns:
@@ -2535,7 +2747,7 @@ def from_arrow(cls, at, index_cols=None, index=None, columns=None):
             columns=new_columns,
             row_lengths=new_lengths,
             column_widths=new_widths,
-            dtypes=pd.Series(data=new_dtypes, index=at.column_names),
+            dtypes=pd.Series(data=new_dtypes, index=dtype_index),
             index_cols=index_cols,
             has_unsupported_data=len(unsupported_cols) > 0,
         )
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py
@@ -274,6 +274,21 @@ def run_exec_plan(cls, plan, index_cols, dtypes, columns):
                         p.frame_id = omniSession.import_pandas_dataframe(obj)
                     else:
                         assert isinstance(obj, pyarrow.Table)
+                        if obj.num_columns == 0:
+                            # Tables without columns are not supported.
+                            # Creating an empty table with index columns only.
+                            idx_names = (
+                                frame._index_cache.names
+                                if frame._index_cache is not None
+                                else [None]
+                            )
+                            idx_names = frame._mangle_index_names(idx_names)
+                            obj = pyarrow.table(
+                                {n: [] for n in idx_names},
+                                schema=pyarrow.schema(
+                                    {n: pyarrow.int64() for n in idx_names}
+                                ),
+                            )
                         p.frame_id = omniSession.import_arrow_table(obj)
 
         calcite_plan = CalciteBuilder().build(plan)


Test patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
@@ -2252,6 +2252,7 @@ def drop_rename_concat(df1, df2, lib, **kwargs):
             drop_rename_concat,
             data=self.data1,
             data2=self.data2,
+            force_lazy=False,
             force_arrow_execute=True,
         )
 
diff --git a/modin/pandas/test/test_concat.py b/modin/pandas/test/test_concat.py
--- a/modin/pandas/test/test_concat.py
+++ b/modin/pandas/test/test_concat.py
@@ -25,12 +25,15 @@
     create_test_dfs,
     default_to_pandas_ignore_string,
 )
-from modin.config import NPartitions
+from modin.config import NPartitions, StorageFormat
 
 NPartitions.put(4)
 
 pytestmark = pytest.mark.filterwarnings(default_to_pandas_ignore_string)
 
+# Initialize env for storage format detection in @pytest.mark.*
+pd.DataFrame()
+
 
 def test_df_concat():
     df, df2 = generate_dfs()
@@ -272,3 +275,56 @@ def test_concat_empty_df_series():
     pdf = pandas.concat((pandas.DataFrame(), pandas.Series([1, 2, 3])))
     mdf = pd.concat((pd.DataFrame(), pd.Series([1, 2, 3])))
     df_equals(pdf, mdf)
+
+
+@pytest.mark.skipif(
+    StorageFormat.get() not in ("Hdk", "Base"),
+    reason="https://github.com/modin-project/modin/issues/5696",
+)
+@pytest.mark.parametrize("col_type", [None, "str"])
+@pytest.mark.parametrize("df1_cols", [0, 90, 100])
+@pytest.mark.parametrize("df2_cols", [0, 90, 100])
+@pytest.mark.parametrize("df1_rows", [0, 100])
+@pytest.mark.parametrize("df2_rows", [0, 100])
+@pytest.mark.parametrize("idx_type", [None, "str"])
+@pytest.mark.parametrize("ignore_index", [True, False])
+@pytest.mark.parametrize("sort", [True, False])
+@pytest.mark.parametrize("join", ["inner", "outer"])
+def test_concat_different_num_cols(
+    col_type,
+    df1_cols,
+    df2_cols,
+    df1_rows,
+    df2_rows,
+    idx_type,
+    ignore_index,
+    sort,
+    join,
+):
+    def create_frame(frame_type, ncols, nrows):
+        def to_str(val):
+            return f"str_{val}"
+
+        off = 0
+        data = {}
+        for n in range(1, ncols + 1):
+            row = range(off + 1, off + nrows + 1)
+            if col_type == "str":
+                row = map(to_str, row)
+            data[f"Col_{n}"] = list(row)
+            off += nrows
+
+        idx = None
+        if idx_type == "str":
+            idx = pandas.Index(map(to_str, range(1, nrows + 1)), name=f"Index_{nrows}")
+        df = frame_type(data=data, index=idx)
+        return df
+
+    def concat(frame_type, lib):
+        df1 = create_frame(frame_type, df1_cols, df1_rows)
+        df2 = create_frame(frame_type, df2_cols, df2_rows)
+        return lib.concat([df1, df2], ignore_index=ignore_index, sort=sort, join=join)
+
+    mdf = concat(pd.DataFrame, pd)
+    pdf = concat(pandas.DataFrame, pandas)
+    df_equals(pdf, mdf)