Repository: modin-project/modin

Issue:
iloc fails with ValueError: cannot reindex from a duplicate axis
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: macOS 10.14.2
- **Modin installed from (source or binary)**: source
- **Modin version**: 0.4
- **Python version**: 3.6.7
- **Exact command to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
When there are duplicate index `modin.dataframe.iloc` fails with `ValueError: cannot reindex from a duplicate axis` while `pandas` works

### Source code / logs
```py
# works
import pandas as pd

samples = pd.DataFrame({'A': [1, 2, 3], 'B': [3, 4, 5]}, index=[0, 0, 1])
samples.iloc[0]

#does not work
import modin.pandas as pd

samples = pd.DataFrame({'A': [1, 2, 3], 'B': [3, 4, 5]}, index=[0, 0, 1])
samples.iloc[0]
```


Discussion:
Thanks @eavidan, great find!

The Indexing methods (`loc`/`iloc`) need to be updated a bit. They work, but they can be slow in some situations, and given the new additions we have been working on we should make sure that this isn't killing performance. 

Since the whole module needs updating, I will take this as my next big undertaking after #522 is merged.

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -2408,9 +2408,9 @@ def view(self, index=None, columns=None):
             np.arange(len(self.columns)), index=self.columns
         )
         if index is not None:
-            index_map_series = index_map_series.reindex(index)
+            index_map_series = index_map_series.iloc[index]
         if columns is not None:
-            column_map_series = column_map_series.reindex(columns)
+            column_map_series = column_map_series.iloc[columns]
         return PandasQueryCompilerView(
             self.data,
             index_map_series.index,
diff --git a/modin/pandas/indexing.py b/modin/pandas/indexing.py
--- a/modin/pandas/indexing.py
+++ b/modin/pandas/indexing.py
@@ -224,9 +224,7 @@ def _broadcast_item(self, row_lookup, col_lookup, item, to_shape):
     def _write_items(self, row_lookup, col_lookup, item):
         """Perform remote write and replace blocks.
         """
-        row_numeric_idx = self.qc.global_idx_to_numeric_idx("row", row_lookup)
-        col_numeric_idx = self.qc.global_idx_to_numeric_idx("col", col_lookup)
-        self.qc.write_items(row_numeric_idx, col_numeric_idx, item)
+        self.qc.write_items(row_lookup, col_lookup, item)
 
 
 class _LocIndexer(_LocationIndexerBase):
@@ -326,17 +324,21 @@ def _compute_lookup(self, row_loc, col_loc) -> Tuple[pandas.Index, pandas.Index]
                 row_loc = [pandas.to_datetime(row_loc[0])]
 
         if isinstance(row_loc, slice):
-            row_lookup = self.qc.index.to_series().loc[row_loc].values
+            row_lookup = self.qc.index.get_indexer_for(
+                self.qc.index.to_series().loc[row_loc]
+            )
         elif isinstance(self.qc.index, pandas.MultiIndex):
-            row_lookup = self.qc.index[self.qc.index.get_locs(row_loc)]
+            row_lookup = self.qc.index.get_locs(row_loc)
         else:
-            row_lookup = self.qc.index[self.qc.index.get_indexer_for(row_loc)]
+            row_lookup = self.qc.index.get_indexer_for(row_loc)
         if isinstance(col_loc, slice):
-            col_lookup = self.qc.columns.to_series().loc[col_loc].values
+            col_lookup = self.qc.columns.get_indexer_for(
+                self.qc.columns.to_series().loc[col_loc]
+            )
         elif isinstance(self.qc.columns, pandas.MultiIndex):
-            col_lookup = self.qc.columns[self.qc.columns.get_locs(col_loc)]
+            col_lookup = self.qc.columns.get_locs(col_loc)
         else:
-            col_lookup = self.qc.columns[self.qc.columns.get_indexer_for(col_loc)]
+            col_lookup = self.qc.columns.get_indexer_for(col_loc)
         return row_lookup, col_lookup
 
 
@@ -361,8 +363,12 @@ def __setitem__(self, key, item):
         super(_iLocIndexer, self).__setitem__(row_lookup, col_lookup, item)
 
     def _compute_lookup(self, row_loc, col_loc) -> Tuple[pandas.Index, pandas.Index]:
-        row_lookup = self.qc.index.to_series().iloc[row_loc].index
-        col_lookup = self.qc.columns.to_series().iloc[col_loc].index
+        row_lookup = (
+            pandas.RangeIndex(len(self.qc.index)).to_series().iloc[row_loc].index
+        )
+        col_lookup = (
+            pandas.RangeIndex(len(self.qc.columns)).to_series().iloc[col_loc].index
+        )
         return row_lookup, col_lookup
 
     def _check_dtypes(self, locator):


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -2524,6 +2524,22 @@ def test_index(self, data):
         pandas_df_cp.index = [str(i) for i in pandas_df_cp.index]
         df_equals(modin_df_cp.index, pandas_df_cp.index)
 
+    @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+    def test_indexing_duplicate_axis(self, data):
+        modin_df = pd.DataFrame(data)
+        pandas_df = pandas.DataFrame(data)
+        modin_df.index = pandas_df.index = [i // 3 for i in range(len(modin_df))]
+        assert any(modin_df.index.duplicated())
+        assert any(pandas_df.index.duplicated())
+
+        df_equals(modin_df.iloc[0], pandas_df.iloc[0])
+        df_equals(modin_df.loc[0], pandas_df.loc[0])
+        df_equals(modin_df.iloc[0, 0:4], pandas_df.iloc[0, 0:4])
+        df_equals(
+            modin_df.loc[0, modin_df.columns[0:4]],
+            pandas_df.loc[0, pandas_df.columns[0:4]],
+        )
+
     def test_info(self):
         data = test_data_values[0]
         with pytest.warns(UserWarning):