Repository: modin-project/modin

Issue:
BUG: repr sometimes selects same column/row twice
porting commit https://github.com/ponder-org/modin-public/commit/d98dec18d0544c0d734dc67635aeb4774c39b8ec

this issue still needs a reproducer


Discussion:
@mvashishtha I don't think this issue actually presents or shows. There is some weirdness with the calculation, but the duplicate column will always disappear during the display phase. I tried multiple different ways and could not reproduce the issue.
It seems the modin code is [by default](https://github.com/modin-project/modin/blob/ac92faef5293c2f8e7e93e5031068c7370e6b668/modin/pandas/dataframe.py#L244) setting num_cols to len(columns) so we don’t run into this issue by default.

however if i mess with the max_columns option I can get it:

```python
import modin.pandas as pd
import pandas
pandas.set_option("display.max_columns", 20)
df = pd.DataFrame([list(range(21))])
print(df)
```

i don’t know if anything appears wrong visually but col_indexer has the duplicate column 10: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] 

and that is a problem when `iloc` on axis 1 is selecting columns from a database.
Should we close this issue @mvashishtha @devin-petersohn?
@RehanSD no, it's still relevant, but we don't need to fix it right now.

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -114,6 +114,53 @@
 _doc_binary_op_kwargs = {"returns": "BasePandasDataset", "left": "BasePandasDataset"}
 
 
+def _get_repr_axis_label_indexer(labels, num_for_repr):
+    """
+    Get the indexer for the given axis labels to be used for the repr.
+
+    Parameters
+    ----------
+    labels : pandas.Index
+        The axis labels.
+    num_for_repr : int
+        The number of elements to display.
+
+    Returns
+    -------
+    slice or list
+        The indexer to use for the repr.
+    """
+    if len(labels) <= num_for_repr:
+        return slice(None)
+    # At this point, the entire axis has len(labels) elements, and num_for_repr <
+    # len(labels). We want to select a pandas subframe containing elements such that:
+    #   - the repr of the pandas subframe will be the same as the repr of the entire
+    #     frame.
+    #   - the pandas repr will not be able to show all the elements and will put an
+    #      ellipsis in the middle
+    #
+    # We accomplish this by selecting some elements from the front and some from the
+    # back, with the front having at most 1 element more than the back. The total
+    # number of elements will be num_for_repr + 1.
+
+    if num_for_repr % 2 == 0:
+        # If num_for_repr is even, take an extra element from the front.
+        # The total number of elements we are selecting is (num_for_repr // 2) * 2 + 1
+        # = num_for_repr + 1
+        front_repr_num = num_for_repr // 2 + 1
+        back_repr_num = num_for_repr // 2
+    else:
+        # If num_for_repr is odd, take an extra element from both the front and the
+        # back. The total number of elements we are selecting is
+        # (num_for_repr // 2) * 2 + 1 + 1 = num_for_repr + 1
+        front_repr_num = num_for_repr // 2 + 1
+        back_repr_num = num_for_repr // 2 + 1
+    all_positions = range(len(labels))
+    return list(all_positions[:front_repr_num]) + (
+        [] if back_repr_num == 0 else list(all_positions[-back_repr_num:])
+    )
+
+
 @_inherit_docstrings(pandas.DataFrame, apilink=["pandas.DataFrame", "pandas.Series"])
 class BasePandasDataset(ClassLogger):
     """
@@ -207,43 +254,9 @@ def _build_repr_df(self, num_rows, num_cols):
                 index=self.index,
                 columns=self.columns if self._is_dataframe else None,
             )
-        if len(self.index) <= num_rows:
-            row_indexer = slice(None)
-        else:
-            # Add one here so that pandas automatically adds the dots
-            # It turns out to be faster to extract 2 extra rows and columns than to
-            # build the dots ourselves.
-            num_rows_for_head = num_rows // 2 + 1
-            num_rows_for_tail = (
-                num_rows_for_head
-                if len(self.index) > num_rows
-                else len(self.index) - num_rows_for_head
-                if len(self.index) - num_rows_for_head >= 0
-                else None
-            )
-            row_indexer = list(range(len(self.index))[:num_rows_for_head]) + (
-                list(range(len(self.index))[-num_rows_for_tail:])
-                if num_rows_for_tail is not None
-                else []
-            )
+        row_indexer = _get_repr_axis_label_indexer(self.index, num_rows)
         if self._is_dataframe:
-            if len(self.columns) <= num_cols:
-                col_indexer = slice(None)
-            else:
-                num_cols_for_front = num_cols // 2 + 1
-                num_cols_for_back = (
-                    num_cols_for_front
-                    if len(self.columns) > num_cols
-                    else len(self.columns) - num_cols_for_front
-                    if len(self.columns) - num_cols_for_front >= 0
-                    else None
-                )
-                col_indexer = list(range(len(self.columns))[:num_cols_for_front]) + (
-                    list(range(len(self.columns))[-num_cols_for_back:])
-                    if num_cols_for_back is not None
-                    else []
-                )
-            indexer = row_indexer, col_indexer
+            indexer = row_indexer, _get_repr_axis_label_indexer(self.columns, num_cols)
         else:
             indexer = row_indexer
         return self.iloc[indexer]._query_compiler.to_pandas()


Test patch:
diff --git a/modin/pandas/test/dataframe/test_iter.py b/modin/pandas/test/dataframe/test_iter.py
--- a/modin/pandas/test/dataframe/test_iter.py
+++ b/modin/pandas/test/dataframe/test_iter.py
@@ -113,9 +113,15 @@ def test___contains__(request, data):
 
 
 @pytest.mark.parametrize("expand_frame_repr", [False, True])
-@pytest.mark.parametrize("max_rows_columns", [(5, 5), (10, 10), (75, 75), (None, None)])
-def test__options_display(max_rows_columns, expand_frame_repr):
-    frame_data = random_state.randint(RAND_LOW, RAND_HIGH, size=(102, 102))
+@pytest.mark.parametrize(
+    "max_rows_columns",
+    [(5, 5), (10, 10), (50, 50), (51, 51), (52, 52), (75, 75), (None, None)],
+)
+@pytest.mark.parametrize("frame_size", [101, 102])
+def test_display_options_for___repr__(max_rows_columns, expand_frame_repr, frame_size):
+    frame_data = random_state.randint(
+        RAND_LOW, RAND_HIGH, size=(frame_size, frame_size)
+    )
     pandas_df = pandas.DataFrame(frame_data)
     modin_df = pd.DataFrame(frame_data)
 
diff --git a/modin/test/storage_formats/base/test_internals.py b/modin/test/storage_formats/base/test_internals.py
--- a/modin/test/storage_formats/base/test_internals.py
+++ b/modin/test/storage_formats/base/test_internals.py
@@ -20,6 +20,7 @@
     df_equals,
 )
 from modin.config import NPartitions
+import modin.pandas as pd
 
 NPartitions.put(4)
 
@@ -93,3 +94,18 @@ def get_reference(df, value, loc):
         # https://github.com/modin-project/modin/issues/5974
         check_dtypes=axis != 0,
     )
+
+
+@pytest.mark.parametrize("num_rows", list(range(1, 5)), ids=lambda x: f"num_rows={x}")
+@pytest.mark.parametrize("num_cols", list(range(1, 5)), ids=lambda x: f"num_cols={x}")
+def test_repr_size_issue_6104(num_rows, num_cols):
+    # this tests an edge case where we used to select exactly num_cols / 2 + 1 columns
+    # from both the front and the back of the dataframe, but the dataframe is such a
+    # length that the front and back columns overlap at one column. The result is that
+    # we convert one column twice to pandas, although we would never see the duplicate
+    # column in the output because pandas would also only represent the num_cols / 2
+    # columns from the front and back.
+    df = pd.DataFrame([list(range(4)) for _ in range(4)])
+    pandas_repr_df = df._build_repr_df(num_rows, num_cols)
+    assert pandas_repr_df.columns.is_unique
+    assert pandas_repr_df.index.is_unique