Repository: modin-project/modin

Issue:
List index out of range when treating a shifting series
### System information
- MacOs Catalina 10.15.6
- modin version: 0.80
- python version: 3.7.5

I'm trying to divide the shifted column of a dataframe for the same column (but not shifted).
Unfortunately I encounter this issue:

```
df = pd.DataFrame({"a":[1, 2 ,3]})
df.shift(-1).a / df.a
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-8-7cc3bcb4fa16> in <module>
      1 df = pd.DataFrame({"a":[1, 2 ,3]})
----> 2 df.shift(-1).a / df.a

~/Envs/strategy/lib/python3.7/site-packages/modin/pandas/series.py in __truediv__(self, right)
    348 
    349     def __truediv__(self, right):
--> 350         return self.truediv(right)
    351 
    352     def __rtruediv__(self, left):

~/Envs/strategy/lib/python3.7/site-packages/modin/pandas/series.py in truediv(self, other, level, fill_value, axis)
   1478         new_self, new_other = self._prepare_inter_op(other)
   1479         return super(Series, new_self).truediv(
-> 1480             new_other, level=level, fill_value=None, axis=axis
   1481         )
   1482 

~/Envs/strategy/lib/python3.7/site-packages/modin/pandas/base.py in truediv(self, other, axis, level, fill_value)
   3299         """
   3300         return self._binary_op(
-> 3301             "truediv", other, axis=axis, level=level, fill_value=fill_value
   3302         )
   3303 

~/Envs/strategy/lib/python3.7/site-packages/modin/pandas/base.py in _binary_op(self, op, other, **kwargs)
    231             )
    232         other = self._validate_other(other, axis, numeric_or_object_only=True)
--> 233         new_query_compiler = getattr(self._query_compiler, op)(other, **kwargs)
    234         return self._create_or_update_from_compiler(new_query_compiler)
    235 

~/Envs/strategy/lib/python3.7/site-packages/modin/data_management/functions/binary_function.py in caller(query_compiler, other, *args, **kwargs)
     49                             lambda x, y: func(x, y, *args, **kwargs),
     50                             other._modin_frame,
---> 51                             join_type=join_type,
     52                         )
     53                     )

~/Envs/strategy/lib/python3.7/site-packages/modin/engines/base/frame/data.py in _binary_op(self, op, right_frame, join_type)
   1604         right_parts = right_parts[0]
   1605         new_frame = self._frame_mgr_cls.binary_operation(
-> 1606             1, left_parts, lambda l, r: op(l, r), right_parts
   1607         )
   1608         new_columns = self.columns.join(right_frame.columns, how=join_type)

~/Envs/strategy/lib/python3.7/site-packages/modin/engines/base/frame/partition_manager.py in binary_operation(cls, axis, left, func, right)
    753                     other_axis_partition=right_partitions[i],
    754                 )
--> 755                 for i in range(len(left_partitions))
    756             ]
    757         )

~/Envs/strategy/lib/python3.7/site-packages/modin/engines/base/frame/partition_manager.py in <listcomp>(.0)
    753                     other_axis_partition=right_partitions[i],
    754                 )
--> 755                 for i in range(len(left_partitions))
    756             ]
    757         )

IndexError: list index out of range
```

The same does not happen when using "standard" pandas:

```
df = pd.DataFrame({"a":[1, 2 ,3]})
df.shift(-1).a / df.a

0    2.0
1    1.5
2    NaN
Name: a, dtype: float64
```


Discussion:
Thanks @gius28 for the report!

We are releasing this week, so this fix will be in the next release.
Looks like problem is in `df.append` method which is used internally in `shift`.

Code to reproduce:
```python
import modin.pandas as pd
import pandas

data = {'a': [0, 1, 2]}
data_high = {'a': [0, 1, 2, 0, 1, 2]}

md_df = pd.DataFrame(data)
pd_df = pandas.DataFrame(data)

md_df_high = pd.DataFrame(data_high)
pd_df_high = pandas.DataFrame(data_high)


md_res = md_df.append(md_df)
pd_res = pd_df.append(pd_df)


print(f'modin res \n{md_res.a / md_df_high.a}')
print(f'pandas res \n{pd_res.a / pd_df_high.a}')
```

Problem is in different partitioning of `md_res` and `md_df_high`. `md_res` consist from 2 partitions with `md_df` in this case. I will see on this.
thanks!

On Mon, Oct 12, 2020 at 4:59 PM Alexey Prutskov <notifications@github.com>
wrote:

> Looks like problem is in df.append method which is used internally in
> shift.
>
> Code to reproduce:
>
> import modin.pandas as pdimport pandas
> data = {'a': [0, 1, 2]}data_high = {'a': [0, 1, 2, 0, 1, 2]}
> md_df = pd.DataFrame(data)pd_df = pandas.DataFrame(data)
> md_df_high = pd.DataFrame(data_high)pd_df_high = pandas.DataFrame(data_high)
>
> md_res = md_df.append(md_df)pd_res = pd_df.append(pd_df)
>
> print(f'modin res \n{md_res.a / md_df_high.a}')print(f'pandas res \n{pd_res.a / pd_df_high.a}')
>
> Problem is in different partitioning of md_res and md_df_high. md_res
> consist from 2 partitions with md_df in this case. I will see on this.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/modin-project/modin/issues/2133#issuecomment-707172296>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AELM5U5BDZBU46JVBYNAN5LSKMKTVANCNFSM4RXEENTA>
> .
>


Resolving patch:
diff --git a/modin/data_management/functions/binary_function.py b/modin/data_management/functions/binary_function.py
--- a/modin/data_management/functions/binary_function.py
+++ b/modin/data_management/functions/binary_function.py
@@ -23,6 +23,7 @@ def call(cls, func, *call_args, **call_kwds):
         def caller(query_compiler, other, *args, **kwargs):
             axis = kwargs.get("axis", 0)
             broadcast = kwargs.pop("broadcast", False)
+            join_type = call_kwds.get("join_type", "outer")
             if isinstance(other, type(query_compiler)):
                 if broadcast:
                     assert (
@@ -39,11 +40,11 @@ def caller(query_compiler, other, *args, **kwargs):
                             axis,
                             lambda l, r: func(l, r.squeeze(), *args, **kwargs),
                             other._modin_frame,
+                            join_type=join_type,
                             preserve_labels=call_kwds.get("preserve_labels", False),
                         )
                     )
                 else:
-                    join_type = call_kwds.get("join_type", "outer")
                     return query_compiler.__constructor__(
                         query_compiler._modin_frame._binary_op(
                             lambda x, y: func(x, y, *args, **kwargs),
diff --git a/modin/engines/base/frame/data.py b/modin/engines/base/frame/data.py
--- a/modin/engines/base/frame/data.py
+++ b/modin/engines/base/frame/data.py
@@ -1402,23 +1402,34 @@ def _apply_select_indices(
                 self._column_widths_cache,
             )
 
-    def broadcast_apply(self, axis, func, other, preserve_labels=True, dtypes=None):
-        """Broadcast partitions of other dataframe partitions and apply a function.
+    def broadcast_apply(
+        self, axis, func, other, join_type="left", preserve_labels=True, dtypes=None
+    ):
+        """
+        Broadcast partitions of other dataframe partitions and apply a function.
 
-        Args:
-            axis: The axis to broadcast over.
-            func: The function to apply.
-            other: The Modin DataFrame to broadcast.
-            preserve_labels: Whether or not to keep labels from this Modin DataFrame.
-            dtypes: "copy" or None. Whether to keep old dtypes or infer new dtypes from
-                data.
+        Parameters
+        ----------
+            axis: int,
+                The axis to broadcast over.
+            func: callable,
+                The function to apply.
+            other: BasePandasFrame
+                The Modin DataFrame to broadcast.
+            join_type: str (optional)
+                The type of join to apply.
+            preserve_labels: boolean (optional)
+                Whether or not to keep labels from this Modin DataFrame.
+            dtypes: "copy" or None (optional)
+                 Whether to keep old dtypes or infer new dtypes from data.
 
-        Returns:
-             A new Modin DataFrame
+        Returns
+        -------
+            BasePandasFrame
         """
         # Only sort the indices if they do not match
         left_parts, right_parts, joined_index = self._copartition(
-            axis, other, "left", sort=not self.axes[axis].equals(other.axes[axis])
+            axis, other, join_type, sort=not self.axes[axis].equals(other.axes[axis])
         )
         # unwrap list returned by `copartition`.
         right_parts = right_parts[0]
@@ -1651,7 +1662,26 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
         """
         if isinstance(other, type(self)):
             other = [other]
-        if all(o.axes[axis].equals(self.axes[axis]) for o in other):
+
+        is_aligning_applied = False
+        for i in range(len(other)):
+            if (
+                len(self._partitions) != len(other[i]._partitions)
+                and len(self.axes[0]) == len(other[i].axes[0])
+                and axis == 0
+            ):
+                is_aligning_applied = True
+                self._partitions = self._frame_mgr_cls.map_axis_partitions(
+                    axis, self._partitions, lambda df: df
+                )
+                other[i]._partitions = other[i]._frame_mgr_cls.map_axis_partitions(
+                    axis, other[i]._partitions, lambda df: df
+                )
+
+        if (
+            all(o.axes[axis].equals(self.axes[axis]) for o in other)
+            and not is_aligning_applied
+        ):
             return (
                 self._partitions,
                 [self._simple_shuffle(axis, o) for o in other],
@@ -1664,8 +1694,13 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
         left_old_idx = self.axes[axis]
         right_old_idxes = index_other_obj
 
+        is_avoid_reindex = len(joined_index) != len(joined_index.unique()) and axis == 0
         # Start with this and we'll repartition the first time, and then not again.
-        if not left_old_idx.equals(joined_index) or force_repartition:
+        if (
+            not is_aligning_applied
+            and not is_avoid_reindex
+            and (force_repartition or not left_old_idx.equals(joined_index))
+        ):
             reindexed_self = self._frame_mgr_cls.map_axis_partitions(
                 axis, self._partitions, lambda df: df.reindex(joined_index, axis=axis)
             )
@@ -1674,7 +1709,11 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
         reindexed_other_list = []
 
         for i in range(len(other)):
-            if right_old_idxes[i].equals(joined_index) and not force_repartition:
+            if (
+                is_aligning_applied
+                or is_avoid_reindex
+                or (not force_repartition and right_old_idxes[i].equals(joined_index))
+            ):
                 reindexed_other = other[i]._partitions
             else:
                 reindexed_other = other[i]._frame_mgr_cls.map_axis_partitions(
@@ -1744,7 +1783,7 @@ def _binary_op(self, op, right_frame, join_type="outer"):
             1, left_parts, lambda l, r: op(l, r), right_parts
         )
         new_columns = self.columns.join(right_frame.columns, how=join_type)
-        return self.__constructor__(new_frame, self.index, new_columns, None, None)
+        return self.__constructor__(new_frame, joined_index, new_columns, None, None)
 
     def _concat(self, axis, others, how, sort):
         """Concatenate this dataframe with one or more others.
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -1637,9 +1637,11 @@ def _prepare_inter_op(self, other):
         """
         if isinstance(other, Series):
             new_self = self.copy()
-            new_self.name = "__reduced__"
             new_other = other.copy()
-            new_other.name = "__reduced__"
+            if self.name == other.name:
+                new_self.name = new_other.name = self.name
+            else:
+                new_self.name = new_other.name = "__reduced__"
         else:
             new_self = self
             new_other = other


Test patch:
diff --git a/modin/pandas/test/dataframe/test_binary.py b/modin/pandas/test/dataframe/test_binary.py
--- a/modin/pandas/test/dataframe/test_binary.py
+++ b/modin/pandas/test/dataframe/test_binary.py
@@ -197,3 +197,62 @@ def test_equals():
         df_equals(modin_df3, modin_df2)
 
     assert modin_df1.equals(modin_df2._query_compiler.to_pandas())
+
+
+@pytest.mark.parametrize("is_more_other_partitions", [True, False])
+@pytest.mark.parametrize(
+    "op_type", ["df_ser", "df_df", "ser_ser_same_name", "ser_ser_different_name"]
+)
+@pytest.mark.parametrize(
+    "is_idx_aligned", [True, False], ids=["idx_aligned", "idx_not_aligned"]
+)
+def test_mismatched_row_partitions(is_idx_aligned, op_type, is_more_other_partitions):
+    data = [0, 1, 2, 3, 4, 5]
+    modin_df1, pandas_df1 = create_test_dfs({"a": data, "b": data})
+    modin_df, pandas_df = modin_df1.loc[:2], pandas_df1.loc[:2]
+
+    modin_df2 = modin_df.append(modin_df)
+    pandas_df2 = pandas_df.append(pandas_df)
+    if is_more_other_partitions:
+        modin_df2, modin_df1 = modin_df1, modin_df2
+        pandas_df2, pandas_df1 = pandas_df1, pandas_df2
+
+    if is_idx_aligned:
+        if is_more_other_partitions:
+            modin_df1.index = pandas_df1.index = pandas_df2.index
+        else:
+            modin_df2.index = pandas_df2.index = pandas_df1.index
+
+    # Pandas don't support this case because result will contain duplicate values by col axis.
+    if op_type == "df_ser" and not is_idx_aligned and is_more_other_partitions:
+        eval_general(
+            modin_df2,
+            pandas_df2,
+            lambda df: df / modin_df1.a
+            if isinstance(df, pd.DataFrame)
+            else df / pandas_df1.a,
+        )
+        return
+
+    if op_type == "df_ser":
+        modin_res = modin_df2 / modin_df1.a
+        pandas_res = pandas_df2 / pandas_df1.a
+    elif op_type == "df_df":
+        modin_res = modin_df2 / modin_df1
+        pandas_res = pandas_df2 / pandas_df1
+    elif op_type == "ser_ser_same_name":
+        modin_res = modin_df2.a / modin_df1.a
+        pandas_res = pandas_df2.a / pandas_df1.a
+    elif op_type == "ser_ser_different_name":
+        modin_res = modin_df2.a / modin_df1.b
+        pandas_res = pandas_df2.a / pandas_df1.b
+    df_equals(modin_res, pandas_res)
+
+
+def test_duplicate_indexes():
+    data = [0, 1, 2, 3, 4, 5]
+    modin_df1, pandas_df1 = create_test_dfs(
+        {"a": data, "b": data}, index=[0, 1, 2, 0, 1, 2]
+    )
+    modin_df2, pandas_df2 = create_test_dfs({"a": data, "b": data})
+    df_equals(modin_df1 / modin_df2, pandas_df1 / pandas_df2)