Repository: modin-project/modin

Issue:
Creating a categorical dataframe with a duplicate column name causes TypeError
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: MacOS Monterey 12.2.1
- **Modin version** (`modin.__version__`): Latest source
- **Python version**: 3.9.9
- **Code we can use to reproduce**:

```python
import modin.pandas as pd
df = pd.DataFrame([["a", "b"]], columns=[0, 0], dtype="category")
print(df)
```

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
Creating a categorical dataframe with a duplicate column name causes `TypeError: all components to combine must be Categorical`. The reason is that [here](https://github.com/modin-project/modin/blob/ee2440c53a1e3bd47736776e7c643f05c4a0db70/modin/core/dataframe/pandas/utils.py#L40) we assume that   calling `df[col]` gives a single column as a `Series`. When `col` is a duplicated column name, `df[col]` instead gives a dataframe.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

Stack trace from the example above:

<details>
<summary>Show summary</summary>

```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [25], in <module>
      1 import modin.pandas as pd
      3 df = pd.DataFrame([["a", "b"]], columns=[0, 0], dtype="category")
----> 4 print(df)

File ~/modin/modin/pandas/base.py:3125, in BasePandasDataset.__str__(self)
   3124 def __str__(self):  # pragma: no cover
-> 3125     return repr(self)

File ~/modin/modin/pandas/dataframe.py:214, in DataFrame.__repr__(self)
    211         i -= 1
    213     num_cols += len(self.columns) - i
--> 214 result = repr(self._build_repr_df(num_rows, num_cols))
    215 if len(self.index) > num_rows or len(self.columns) > num_cols:
    216     # The split here is so that we don't repr pandas row lengths.
    217     return result.rsplit("\n\n", 1)[0] + "\n\n[{0} rows x {1} columns]".format(
    218         len(self.index), len(self.columns)
    219     )

File ~/modin/modin/pandas/base.py:198, in BasePandasDataset._build_repr_df(self, num_rows, num_cols)
    196 else:
    197     indexer = row_indexer
--> 198 return self.iloc[indexer]._query_compiler.to_pandas()

File ~/modin/modin/core/storage_formats/pandas/query_compiler.py:257, in PandasQueryCompiler.to_pandas(self)
    256 def to_pandas(self):
--> 257     return self._modin_frame.to_pandas()

File ~/modin/modin/core/dataframe/pandas/dataframe/dataframe.py:114, in lazy_metadata_decorator.<locals>.decorator.<locals>.run_f_on_minimally_updated_metadata(self, *args, **kwargs)
    112     elif apply_axis == "rows":
    113         obj._propagate_index_objs(axis=0)
--> 114 result = f(self, *args, **kwargs)
    115 if apply_axis is None and not transpose:
    116     result._deferred_index = self._deferred_index

File ~/modin/modin/core/dataframe/pandas/dataframe/dataframe.py:2760, in PandasDataframe.to_pandas(self)
   2751 @lazy_metadata_decorator(apply_axis="both")
   2752 def to_pandas(self):
   2753     """
   2754     Convert this Modin DataFrame to a pandas DataFrame.
   2755
   (...)
   2758     pandas.DataFrame
   2759     """
-> 2760     df = self._partition_mgr_cls.to_pandas(self._partitions)
   2761     if df.empty:
   2762         df = pandas.DataFrame(columns=self.columns, index=self.index)

File ~/modin/modin/core/dataframe/pandas/partitioning/partition_manager.py:651, in PandasDataframePartitionManager.to_pandas(cls, partitions)
    649     return pandas.DataFrame()
    650 else:
--> 651     return concatenate(df_rows)

File ~/modin/modin/core/dataframe/pandas/utils.py:40, in concatenate(dfs)
     35 categoricals_columns = set.intersection(
     36     *[set(df.select_dtypes("category").columns.tolist()) for df in dfs]
     37 )
     39 for col in categoricals_columns:
---> 40     uc = union_categoricals([df[col] for df in dfs])
     41     for df in dfs:
     42         df[col] = pandas.Categorical(df[col], categories=uc.categories)

File /usr/local/lib/python3.9/site-packages/pandas/core/dtypes/concat.py:280, in union_categoricals(to_union, sort_categories, ignore_order)
    277     else:
    278         raise TypeError("all components to combine must be Categorical")
--> 280 to_union = [_maybe_unwrap(x) for x in to_union]
    281 first = to_union[0]
    283 if not all(
    284     is_dtype_equal(other.categories.dtype, first.categories.dtype)
    285     for other in to_union[1:]
    286 ):

File /usr/local/lib/python3.9/site-packages/pandas/core/dtypes/concat.py:280, in <listcomp>(.0)
    277     else:
    278         raise TypeError("all components to combine must be Categorical")
--> 280 to_union = [_maybe_unwrap(x) for x in to_union]
    281 first = to_union[0]
    283 if not all(
    284     is_dtype_equal(other.categories.dtype, first.categories.dtype)
    285     for other in to_union[1:]
    286 ):

File /usr/local/lib/python3.9/site-packages/pandas/core/dtypes/concat.py:278, in union_categoricals.<locals>._maybe_unwrap(x)
    276     return x
    277 else:
--> 278     raise TypeError("all components to combine must be Categorical")

TypeError: all components to combine must be Categorical
```

</details>



Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/pandas/utils.py b/modin/core/dataframe/pandas/utils.py
--- a/modin/core/dataframe/pandas/utils.py
+++ b/modin/core/dataframe/pandas/utils.py
@@ -22,6 +22,8 @@ def concatenate(dfs):
     """
     Concatenate pandas DataFrames with saving 'category' dtype.
 
+    All dataframes' columns must be equal to each other.
+
     Parameters
     ----------
     dfs : list
@@ -32,13 +34,15 @@ def concatenate(dfs):
     pandas.DataFrame
         A pandas DataFrame.
     """
-    categoricals_columns = set.intersection(
-        *[set(df.select_dtypes("category").columns.tolist()) for df in dfs]
-    )
-
-    for col in categoricals_columns:
-        uc = union_categoricals([df[col] for df in dfs])
+    for df in dfs:
+        assert df.columns.equals(dfs[0].columns)
+    for i in range(len(dfs[0].columns)):
+        if dfs[0].dtypes.iloc[i].name != "category":
+            continue
+        columns = [df.iloc[:, i] for df in dfs]
+        union = union_categoricals(columns)
         for df in dfs:
-            df[col] = pandas.Categorical(df[col], categories=uc.categories)
-
+            df.iloc[:, i] = pandas.Categorical(
+                df.iloc[:, i], categories=union.categories
+            )
     return pandas.concat(dfs)


Test patch:
diff --git a/modin/pandas/test/test_general.py b/modin/pandas/test/test_general.py
--- a/modin/pandas/test/test_general.py
+++ b/modin/pandas/test/test_general.py
@@ -19,6 +19,7 @@
 from numpy.testing import assert_array_equal
 from modin.utils import get_current_execution, to_pandas
 from modin.test.test_utils import warns_that_defaulting_to_pandas
+from pandas.testing import assert_frame_equal
 
 from .utils import (
     test_data_values,
@@ -730,6 +731,32 @@ def test_to_pandas_indices(data):
         ), f"Levels of indices at axis {axis} are different!"
 
 
+def test_create_categorical_dataframe_with_duplicate_column_name():
+    # This tests for https://github.com/modin-project/modin/issues/4312
+    pd_df = pandas.DataFrame(
+        {
+            "a": pandas.Categorical([1, 2]),
+            "b": [4, 5],
+            "c": pandas.Categorical([7, 8]),
+        }
+    )
+    pd_df.columns = ["a", "b", "a"]
+    md_df = pd.DataFrame(pd_df)
+    # Use assert_frame_equal instead of the common modin util df_equals because
+    # we should check dtypes of the new categorical with check_dtype=True.
+    # TODO(https://github.com/modin-project/modin/issues/3804): Make
+    # df_equals set check_dtype=True and use df_equals instead.
+    assert_frame_equal(
+        md_df._to_pandas(),
+        pd_df,
+        check_dtype=True,
+        check_index_type=True,
+        check_column_type=True,
+        check_names=True,
+        check_categorical=True,
+    )
+
+
 @pytest.mark.skipif(
     get_current_execution() != "BaseOnPython",
     reason="This test make sense only on BaseOnPython execution.",
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -496,13 +496,16 @@ def df_categories_equals(df1, df2):
         else:
             return True
 
-    df1_categorical_columns = df1.select_dtypes(include="category").columns
-    df2_categorical_columns = df2.select_dtypes(include="category").columns
-    assert df1_categorical_columns.equals(df2_categorical_columns)
-    for column in df1_categorical_columns:
+    df1_categorical = df1.select_dtypes(include="category")
+    df2_categorical = df2.select_dtypes(include="category")
+    assert df1_categorical.columns.equals(df2_categorical.columns)
+    # Use an index instead of a column name to iterate through columns. There
+    # may be duplicate colum names. e.g. if two columns are named col1,
+    # selecting df1_categorical["col1"] gives a dataframe of width 2 instead of a series.
+    for i in range(len(df1_categorical.columns)):
         assert_extension_array_equal(
-            df1[column].values,
-            df2[column].values,
+            df1_categorical.iloc[:, i].values,
+            df2_categorical.iloc[:, i].values,
             check_dtype=False,
         )