Repository: modin-project/modin

Issue:
`read_csv` with OmniSci backend doesn't handle duplicated columns
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 20.04
- **Modin version**: 563f26d031d3b50b02d1b7a1540dc11a009350cc
- **Python version**: 3.7.10
- **Code we can use to reproduce**:
```
import os

os.environ["MODIN_EXPERIMENTAL"] = "True"
os.environ["MODIN_STORAGE_FORMAT"] = "hdk"

import modin.pandas as pd

test_filename = "test_csv.csv"

data = "col,col,col,col\n5, 6, 7, 8\n9, 10, 11, 12\n"


try:
    with open(test_filename, "w") as f:
        f.write(data)

    df_modin = pd.read_csv(test_filename)
    print(df_modin)
finally:
    os.remove(test_filename)
```

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```
Traceback (most recent call last):
  File "test.py", line 487, in <module>
    print(df_modin)
  File "/modin/modin/pandas/base.py", line 2804, in __str__
    return repr(self)
  File "/modin/modin/pandas/dataframe.py", line 183, in __repr__
    result = repr(self._build_repr_df(num_rows, num_cols))
  File "/modin/modin/pandas/base.py", line 202, in _build_repr_df
    return self.iloc[indexer]._query_compiler.to_pandas()
  File "/modin/modin/pandas/indexing.py", line 730, in __getitem__
    result = super(_iLocIndexer, self).__getitem__(row_lookup, col_lookup, ndim)
  File "/modin/modin/pandas/indexing.py", line 276, in __getitem__
    qc_view = self.qc.view(row_lookup, col_lookup)
  File "/modin/modin/experimental/backends/omnisci/query_compiler.py", line 46, in method_wrapper
    return method(self, *args, **kwargs)
  File "/modin/modin/experimental/backends/omnisci/query_compiler.py", line 166, in view
    self._modin_frame.mask(row_numeric_idx=index, col_numeric_idx=columns)
  File "/modin/modin/experimental/engines/omnisci_on_ray/frame/data.py", line 181, in mask
    force_execution_mode=self._force_execution_mode,
  File "/modin/modin/experimental/engines/omnisci_on_ray/frame/data.py", line 98, in __init__
    ), f"unaligned dtypes ({dtypes}) and table columns ({self._table_cols})"
AssertionError: unaligned dtypes ([col    int64
col    int64
col    int64
col    int64
dtype: object]) and table columns (['col', 'col', 'col', 'col'])
```



Discussion:
Hi @amyskov! Is this issue still present on master?
@AndreyPavlenko please check this one
  The issue is reproducible. It fails when converting pyarrow table to pandas:
```
File "modin/modin/core/dataframe/pandas/partitioning/partition_manager.py", line 644, in <listcomp>
    retrieved_objects = [[obj.to_pandas() for obj in part] for part in partitions]
  File "modin/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py", line 85, in to_pandas
    return obj.to_pandas()
  File "pyarrow/array.pxi", line 766, in pyarrow.lib._PandasConvertible.to_pandas
  File "pyarrow/table.pxi", line 1815, in pyarrow.lib.Table._to_pandas
  File "python3.8/site-packages/pyarrow/pandas_compat.py", line 788, in table_to_blockmanager
    columns = _deserialize_column_index(table, all_columns, column_indexes)
  File "python3.8/site-packages/pyarrow/pandas_compat.py", line 903, in _deserialize_column_index
    columns = _flatten_single_level_multiindex(columns)
  File "python3.8/site-packages/pyarrow/pandas_compat.py", line 1144, in _flatten_single_level_multiindex
    raise ValueError('Found non-unique column index')
ValueError: Found non-unique column index
```

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
@@ -300,6 +300,26 @@ def read_csv(
                 convert_options=co,
             )
 
+            col_names = at.column_names
+            col_counts = {}
+            for name in col_names:
+                col_counts[name] = 1 if name in col_counts else 0
+
+            if len(col_names) != len(col_counts):
+                for i, name in enumerate(col_names):
+                    count = col_counts[name]
+                    if count != 0:
+                        if count == 1:
+                            col_counts[name] = 2
+                        else:
+                            new_name = f"{name}.{count - 1}"
+                            while new_name in col_counts:
+                                new_name = f"{name}.{count}"
+                                count += 1
+                            col_counts[name] = count + 1
+                            col_names[i] = new_name
+                at = at.rename_columns(col_names)
+
             return cls.from_arrow(at)
         except (
             pa.ArrowNotImplementedError,


Test patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
@@ -18,6 +18,8 @@
 import pytest
 import re
 
+from pandas._testing import ensure_clean
+
 from modin.config import StorageFormat
 from modin.pandas.test.utils import (
     io_ops_bad_exc,
@@ -346,6 +348,32 @@ def test_read_csv_col_handling(
             usecols=usecols,
         )
 
+    @pytest.mark.parametrize(
+        "cols",
+        [
+            "c1,c2,c3",
+            "c1,c1,c2",
+            "c1,c1,c1.1,c1.2,c1",
+            "c1,c1,c1,c1.1,c1.2,c1.3",
+            "c1.1,c1.2,c1.3,c1,c1,c1",
+            "c1.1,c1,c1.2,c1,c1.3,c1",
+            "c1,c1.1,c1,c1.2,c1,c1.3",
+            "c1,c1,c1.1,c1.1,c1.2,c2",
+            "c1,c1,c1.1,c1.1,c1.2,c1.2,c2",
+            "c1.1,c1.1,c1,c1,c1.2,c1.2,c2",
+            "c1.1,c1,c1.1,c1,c1.1,c1.2,c1.2,c2",
+        ],
+    )
+    def test_read_csv_duplicate_cols(self, cols):
+        def test(df, lib, **kwargs):
+            data = f"{cols}\n"
+            with ensure_clean(".csv") as fname:
+                with open(fname, "w") as f:
+                    f.write(data)
+                return lib.read_csv(fname)
+
+        run_and_compare(test, data={})
+
 
 class TestMasks:
     data = {
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -376,17 +376,6 @@ def test_read_csv_parsing_2(
         names,
         encoding,
     ):
-        xfail_case = (
-            StorageFormat.get() == "Hdk"
-            and header is not None
-            and isinstance(skiprows, int)
-            and names is None
-            and nrows is None
-        )
-        if xfail_case:
-            pytest.xfail(
-                "read_csv fails because of duplicated columns names - issue #3080"
-            )
         if request.config.getoption(
             "--simulate-cloud"
         ).lower() != "off" and is_list_like(skiprows):
@@ -417,6 +406,7 @@ def test_read_csv_parsing_2(
                     pytest.xfail(
                         "read_csv incorrect output with float data - issue #2634"
                     )
+
             eval_io(
                 fn_name="read_csv",
                 raising_exceptions=None,
@@ -491,10 +481,6 @@ def test_read_csv_squeeze(self, request, test_case):
             )
 
     def test_read_csv_mangle_dupe_cols(self):
-        if StorageFormat.get() == "Hdk":
-            pytest.xfail(
-                "processing of duplicated columns in HDK storage format is not supported yet - issue #3080"
-            )
         with ensure_clean() as unique_filename, pytest.warns(
             FutureWarning, match="'mangle_dupe_cols' keyword is deprecated"
         ):
@@ -734,6 +720,21 @@ def test_read_csv_file_format(
                     line_terminator=lineterminator,
                 )
 
+            if (
+                (StorageFormat.get() == "Hdk")
+                and (escapechar is not None)
+                and (lineterminator is None)
+                and (thousands is None)
+                and (decimal == ".")
+            ):
+                with open(unique_filename, "r") as f:
+                    if any(
+                        line.find(f',"{escapechar}') != -1 for _, line in enumerate(f)
+                    ):
+                        pytest.xfail(
+                            "Tests with this character sequence fail due to #5649"
+                        )
+
             eval_io(
                 raising_exceptions=None,
                 fn_name="read_csv",
@@ -987,13 +988,6 @@ def test_read_csv_s3_issue4658(self):
     @pytest.mark.parametrize("names", [list("XYZ"), None])
     @pytest.mark.parametrize("skiprows", [1, 2, 3, 4, None])
     def test_read_csv_skiprows_names(self, names, skiprows):
-        if StorageFormat.get() == "Hdk" and names is None and skiprows in [1, None]:
-            # If these conditions are satisfied, columns names will be inferred
-            # from the first row, that will contain duplicated values, that is
-            # not supported by  `HDK` storage format yet.
-            pytest.xfail(
-                "processing of duplicated columns in HDK storage format is not supported yet - issue #3080"
-            )
         eval_io(
             fn_name="read_csv",
             # read_csv kwargs