Repository: modin-project/modin

Issue:
Inserts new Series objects failed with "Internal Error"
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: `Windows 10`
- **Modin version** (`modin.__version__`): `0.8.1.1+36.ga7d3093`
- **Python version**: `Python 3.7.8`
- **Code we can use to reproduce**:

```python
import modin.pandas as mpd
# import pandas as mpd
import numpy as np
import datetime
from pytz import timezone
import os

print(mpd.__version__)

mpd.DEFAULT_NPARTITIONS = 4


def line_count(fname):
    line_count = 0
    with open(fname) as ff:
        for line in ff:
            line_count += 1
    return line_count


def timestamp_to_local(ts, tzone="Australia/Melbourne"):
    """
    Convert unixtimestamp to local time. Output format:year-month-day hour:minute:seconds

    Args:
        ts: unix timestamp
        tzone: Time zone

    Return:
        local: local timestamp
    """

    local = (
        timezone("UTC")
        .localize(datetime.datetime.utcfromtimestamp(ts))
        .astimezone(timezone(tzone))
    )
    local = local.strftime(format="%Y-%m-%d %H:%M:%S:%f")

    return local


fpath = "data"
# ID='PJ'
session = "A018B6_200330-043359"
E4_path = os.path.join(fpath, session)
tzone = "Australia/Melbourne"

prev_end = 0
fs0 = 32

tempfp = os.path.join(E4_path, "TEMP.csv")


prev_endm = 0
accfp = "data/A018B6_200330-043359/ACC.csv"
accm = mpd.read_csv(accfp, header=None)
startm = accm.iloc[0][0]
tm = np.linspace(startm, startm + (len(accm) - 1) / fs0, len(accm))
accm = accm.iloc[2:, :]
accm.reset_index(drop=True, inplace=True)
accm.columns = ["X", "Y", "Z"]
tempm = mpd.read_csv(tempfp, header=None, skiprows=2)
tempm = tempm.loc[tempm.index.repeat(8)].reset_index(drop=True)[0].values
accm["X"] = accm["X"] / 64
accm["Y"] = accm["Y"] / 64
accm["Z"] = accm["Z"] / 64
accm.reset_index(drop=True, inplace=True)
accm["Time"] = mpd.Series(tm)

accm = accm.iloc[
    accm[(accm.X.notnull()) & (accm.Y.notnull()) & (accm.Z.notnull())].index.values, :
]
accm["Time"] = accm["Time"].apply(timestamp_to_local, tzone=tzone)
accm.reset_index(drop=True, inplace=True)
accm["T"] = mpd.Series(tempm)
print(accm)

```

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

Initial found in https://discuss.modin.org/t/modin-errors-out-on-pytz-timezone/119.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

```sh
Traceback (most recent call last):
  File "reproducer_timezone.py", line 77, in <module>
    print(accm)
  File "...\modin\pandas\base.py", line 2684, in __str__
    return repr(self)
  File "...\modin\pandas\dataframe.py", line 182, in __repr__
    result = repr(self._build_repr_df(num_rows, num_cols))
  File "...\modin\pandas\base.py", line 168, in _build_repr_df
    return self.iloc[indexer]._query_compiler.to_pandas()
  File "...\modin\backends\pandas\query_compiler.py", line 231, in to_pandas
    return self._modin_frame.to_pandas()
  File "...\modin\engines\base\frame\data.py", line 1956, in to_pandas
    f"Internal and external indices on axis {axis} do not match.",
  File "...\modin\error_message.py", line 63, in catch_bugs_and_request_email
    " caused this error.\n{}".format(extra_log)
Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Internal and external indices on axis 0 do not match.
```

<details>
<summary>TEMP.csv</summary>

```csv
24.67
24.69
24.69
24.69
24.69
24.65
24.65
24.65
24.65
24.65
24.65
24.65
24.65
24.69
24.69
24.69
24.69
24.65
24.65
24.65
24.65
24.63
24.63
24.63
24.63
```
</details>

<details>
<summary>ACC.csv</summary>

```csv
-22,59,14
-26,43,20
-23,52,23
-24,56,25
-25,52,30
-26,55,34
-26,58,28
-29,58,23
-30,37,23
-28,51,26
-28,57,25
-29,47,25
-32,53,36
-29,50,32
-31,43,27
-39,49,32
-41,50,28
-41,39,29
-42,34,33
-45,44,36
-40,39,26
-40,28,27
-42,37,34
-41,37,32
-35,38,22
-40,41,26
-41,49,30
-39,51,30
-36,47,24
-36,45,22
-37,46,25
-38,51,27
-36,44,20
-37,46,27
-35,47,25
-36,45,20
-38,49,25
-37,48,21
-32,48,12
-37,50,30
-36,45,22
-36,49,24
-38,49,17
-37,49,25
-36,45,23
-35,42,20
-37,47,34
-36,49,26
-36,45,22
-37,51,22
-39,55,20
-41,58,19
-38,56,20
-38,54,26
-37,53,24
-35,53,28
-30,51,30
-26,43,27
-21,48,19
-23,55,21
-23,58,23
-19,56,19
-18,58,23
-17,54,19
-16,59,19
-14,61,19
-8,61,17
-5,62,16
-2,55,23
4,56,28
17,72,40
21,45,44
19,24,25
10,40,22
9,50,27
6,63,41
10,49,37
13,41,64
22,39,50
-2,54,50
-9,45,30
7,56,32
9,59,39
18,58,24
26,51,5
30,51,1
47,51,-12
51,52,-18
43,41,-25
35,30,-35
51,31,-35
59,28,-43
49,20,-39
53,27,-29
44,21,-24
44,27,-11
42,30,-1
56,43,1
50,37,-8
49,34,-9
48,42,-2
48,56,16
39,55,19
30,56,20
30,58,25
17,61,20
27,63,25
37,29,14
27,42,6
35,61,24
31,63,31
26,63,21
16,46,21
8,39,20
-6,56,23
-7,52,28
-25,60,41
-28,62,14
-27,45,-27
-19,43,30
-25,89,-1
74,118,-104
-17,88,-39
-4,31,-26
-15,98,-23
-6,107,55
2,83,-7
-5,64,-4
-6,66,6
-1,59,-6
-3,44,-19
-21,59,-21
-25,61,-28
-32,58,-27
-32,58,-24
-27,54,27
-36,86,-25
-30,58,-16
-29,65,-15
-27,66,-16
-20,54,-14
-19,53,-10
-22,67,-12
-19,62,-12
-21,60,-13
-20,59,-14
-26,57,-17
-17,41,-15
-24,53,-22
-22,67,16
-27,62,-19
-27,50,-25
-31,60,-24
-30,61,-15
-27,60,-18
-21,50,-11
-21,54,-9
-25,59,-9
-26,60,-10
-32,79,-11
-28,73,-10
-23,57,-10
-19,44,-8
-20,45,-10
-21,47,-12
-27,72,-18
-26,63,-16
-25,56,-19
-26,46,16
-27,46,-19
-23,44,-14
-26,55,-15
-34,88,-13
-32,88,-14
-22,58,-8
-21,58,-7
-28,65,-4
-26,57,-11
-28,55,-16
-30,57,-16
-28,53,-16
-26,54,-11
-29,64,-14
-25,57,-13
-22,51,-13
-31,71,-16
-30,74,-17
-23,66,-13
-15,47,-15
-15,55,-15
-11,47,-17
-7,35,-20
-13,49,-19
-20,61,-17
-26,79,-18
-25,64,-16
-28,66,-17
-28,66,-17
-18,72,-13
-14,63,-14
-9,54,-12
-12,62,-15
2,75,-62
-7,59,3
-10,60,-12
-18,69,-13
-16,62,-11
-18,51,-18
-21,54,-20
-29,73,-20
-28,72,-20
-21,61,-16
-20,58,-14
-16,53,-10
-15,55,-9
-20,69,-11
-22,73,-10
-18,65,-7
-2,62,-18
-14,63,0
-14,51,-12
-11,41,-15
```
</details>


Discussion:
This issue has been mentioned on **Modin Discuss**. There might be relevant details there:

https://discuss.modin.org/t/modin-errors-out-on-pytz-timezone/119/51

Inconsistent partitions for `T` column:

```python
(Pdb)  [obj.length() for obj in self._partitions.T[0]] # X 
[56, 56, 56, 52]
(Pdb)  [obj.length() for obj in self._partitions.T[1]] # Y
[56, 56, 56, 52]
(Pdb)  [obj.length() for obj in self._partitions.T[2]] # Z
[56, 56, 56, 52]
(Pdb)  [obj.length() for obj in self._partitions.T[3]] # Time
[56, 56, 56, 52]
(Pdb)  [obj.length() for obj in self._partitions.T[4]] # T
[55, 55, 55, 55]
```
This issue has been mentioned on **Modin Discuss**. There might be relevant details there:

https://discuss.modin.org/t/modin-errors-out-on-pytz-timezone/119/60


Resolving patch:
diff --git a/modin/engines/base/frame/data.py b/modin/engines/base/frame/data.py
--- a/modin/engines/base/frame/data.py
+++ b/modin/engines/base/frame/data.py
@@ -1646,6 +1646,8 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
         """
         Copartition two dataframes.
 
+        Perform aligning of partitions, index and partition blocks.
+
         Parameters
         ----------
             axis : 0 or 1
@@ -1694,6 +1696,7 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
                 [self._simple_shuffle(axis, o) for o in other],
                 self.axes[axis].copy(),
             )
+
         index_other_obj = [o.axes[axis] for o in other]
         joined_index = self._join_index_objects(axis, index_other_obj, how, sort)
         # We have to set these because otherwise when we perform the functions it may
@@ -1701,32 +1704,45 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
         left_old_idx = self.axes[axis]
         right_old_idxes = index_other_obj
 
-        is_avoid_reindex = len(joined_index) != len(joined_index.unique()) and axis == 0
+        def make_map_func():
+            if not joined_index.is_unique and axis == 0:
+                return lambda df: df
+            return lambda df: df.reindex(joined_index, axis=axis)
+
         # Start with this and we'll repartition the first time, and then not again.
-        if (
-            not is_aligning_applied
-            and not is_avoid_reindex
-            and (force_repartition or not left_old_idx.equals(joined_index))
+        if is_aligning_applied or (
+            not force_repartition and left_old_idx.equals(joined_index)
         ):
+            reindexed_self = self._partitions
+        else:
             reindexed_self = self._frame_mgr_cls.map_axis_partitions(
-                axis, self._partitions, lambda df: df.reindex(joined_index, axis=axis)
+                axis,
+                self._partitions,
+                make_map_func(),
             )
-        else:
-            reindexed_self = self._partitions
-        reindexed_other_list = []
 
+        def get_column_widths(partitions):
+            if len(partitions) > 0:
+                return [obj.width() for obj in partitions[0]]
+
+        def get_row_lengths(partitions):
+            if len(partitions.T) > 0:
+                return [obj.length() for obj in partitions.T[0]]
+
+        reindexed_other_list = []
         for i in range(len(other)):
-            if (
-                is_aligning_applied
-                or is_avoid_reindex
-                or (not force_repartition and right_old_idxes[i].equals(joined_index))
+            if is_aligning_applied or (
+                not force_repartition and right_old_idxes[i].equals(joined_index)
             ):
                 reindexed_other = other[i]._partitions
             else:
                 reindexed_other = other[i]._frame_mgr_cls.map_axis_partitions(
                     axis,
                     other[i]._partitions,
-                    lambda df: df.reindex(joined_index, axis=axis),
+                    make_map_func(),
+                    lengths=get_row_lengths(reindexed_self)
+                    if axis == 0
+                    else get_column_widths(reindexed_self),
                 )
             reindexed_other_list.append(reindexed_other)
         return reindexed_self, reindexed_other_list, joined_index
diff --git a/modin/engines/base/frame/partition_manager.py b/modin/engines/base/frame/partition_manager.py
--- a/modin/engines/base/frame/partition_manager.py
+++ b/modin/engines/base/frame/partition_manager.py
@@ -214,22 +214,25 @@ def broadcast_axis_partitions(
         left,
         right,
         keep_partitioning=False,
+        lengths=None,
     ):
         """
         Broadcast the right partitions to left and apply a function along full axis.
 
         Parameters
         ----------
-            axis : The axis to apply and broadcast over.
-            apply_func : The function to apply.
-            left : The left partitions.
-            right : The right partitions.
-            keep_partitioning : boolean. Default is False
-                The flag to keep partitions for Modin Frame.
+        axis : The axis to apply and broadcast over.
+        apply_func : The function to apply.
+        left : The left partitions.
+        right : The right partitions.
+        keep_partitioning : boolean. Default is False
+            The flag to keep partitions for Modin Frame.
+        lengths : list(int)
+            The list of lengths to shuffle the object.
 
         Returns
         -------
-            A new `np.array` of partition objects.
+        A new `np.array` of partition objects.
         """
         # Since we are already splitting the DataFrame back up after an
         # operation, we will just use this time to compute the number of
@@ -245,12 +248,19 @@ def broadcast_axis_partitions(
         # may want to line to partitioning up with another BlockPartitions object. Since
         # we don't need to maintain the partitioning, this gives us the opportunity to
         # load-balance the data as well.
+        kw = {
+            "num_splits": num_splits,
+            "other_axis_partition": right_partitions,
+        }
+        if lengths:
+            kw["_lengths"] = lengths
+            kw["manual_partition"] = True
+
         result_blocks = np.array(
             [
                 part.apply(
                     preprocessed_map_func,
-                    num_splits=num_splits,
-                    other_axis_partition=right_partitions,
+                    **kw,
                 )
                 for part in left_partitions
             ]
@@ -295,20 +305,23 @@ def map_axis_partitions(
         partitions,
         map_func,
         keep_partitioning=False,
+        lengths=None,
     ):
         """
         Applies `map_func` to every partition.
 
         Parameters
         ----------
-            axis : 0 or 1
-                The axis to perform the map across (0 - index, 1 - columns).
-            partitions : NumPy array
-                The partitions of Modin Frame.
-            map_func : callable
-                The function to apply.
-            keep_partitioning : boolean. Default is False
-                The flag to keep partitions for Modin Frame.
+        axis : 0 or 1
+            The axis to perform the map across (0 - index, 1 - columns).
+        partitions : NumPy array
+            The partitions of Modin Frame.
+        map_func : callable
+            The function to apply.
+        keep_partitioning : bool. Default is False
+            The flag to keep partitions for Modin Frame.
+        lengths : list(int)
+            The list of lengths to shuffle the object.
 
         Returns
         -------
@@ -326,6 +339,7 @@ def map_axis_partitions(
             apply_func=map_func,
             keep_partitioning=keep_partitioning,
             right=None,
+            lengths=lengths,
         )
 
     @classmethod
diff --git a/modin/engines/ray/pandas_on_ray/frame/axis_partition.py b/modin/engines/ray/pandas_on_ray/frame/axis_partition.py
--- a/modin/engines/ray/pandas_on_ray/frame/axis_partition.py
+++ b/modin/engines/ray/pandas_on_ray/frame/axis_partition.py
@@ -33,6 +33,7 @@ def __init__(self, list_of_blocks):
     def deploy_axis_func(
         cls, axis, func, num_splits, kwargs, maintain_partitioning, *partitions
     ):
+        lengths = kwargs.get("_lengths", None)
         return deploy_ray_func._remote(
             args=(
                 PandasFrameAxisPartition.deploy_axis_func,
@@ -43,7 +44,7 @@ def deploy_axis_func(
                 maintain_partitioning,
             )
             + tuple(partitions),
-            num_returns=num_splits * 3,
+            num_returns=num_splits * 3 if lengths is None else len(lengths) * 3,
         )
 
     @classmethod


Test patch:
diff --git a/modin/test/backends/pandas/test_internals.py b/modin/test/backends/pandas/test_internals.py
new file mode 100644
--- /dev/null
+++ b/modin/test/backends/pandas/test_internals.py
@@ -0,0 +1,40 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+import modin.pandas as pd
+
+
+def test_aligning_blocks():
+    # Test problem when modin frames have the same number of rows, but different
+    # blocks (partition.list_of_blocks). See #2322 for details
+    accm = pd.DataFrame(["-22\n"] * 162)
+    accm = accm.iloc[2:, :]
+    accm.reset_index(drop=True, inplace=True)
+    accm["T"] = pd.Series(["24.67\n"] * 145)
+
+    # see #2322 for details
+    repr(accm)
+
+
+def test_aligning_blocks_with_duplicated_index():
+    # Same problem as in `test_aligning_blocks` but with duplicated values in index.
+    data11 = [0, 1]
+    data12 = [2, 3]
+
+    data21 = [0]
+    data22 = [1, 2, 3]
+
+    df1 = pd.DataFrame(data11).append(pd.DataFrame(data12))
+    df2 = pd.DataFrame(data21).append(pd.DataFrame(data22))
+
+    repr(df1 - df2)