Repository: modin-project/modin

Issue:
`__setitem__` after `read_csv` with a `Series` from pandas causes metadata error

### Describe the problem
<!-- Describe the problem clearly here. -->
`__setitem__` after `read_csv` with a `Series` from pandas causes metadata error

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```python
In [1]: import modin.pandas as pd                                                                                                                                              
df ^[[A
In [2]: df = pd.read_csv("test_1mx256.csv")                                                                                                                                    

In [3]: df["new"] = pd.Series(list(range(len(df))))                                                                                                                            
```

Error
```
UserWarning: Distributing <class 'list'> object. This may take some time.

In [4]: df                                                                                                                                                                     
Out[4]: ---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/IPython/core/formatters.py in __call__(self, obj)
    700                 type_pprinters=self.type_printers,
    701                 deferred_pprinters=self.deferred_printers)
--> 702             printer.pretty(obj)
    703             printer.flush()
    704             return stream.getvalue()

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/IPython/lib/pretty.py in pretty(self, obj)
    400                         if cls is not object \
    401                                 and callable(cls.__dict__.get('__repr__')):
--> 402                             return _repr_pprint(obj, self, cycle)
    403 
    404             return _default_pprint(obj, self, cycle)

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
    695     """A pprint that just redirects to the normal repr function."""
    696     # Find newlines and replace them with p.break_()
--> 697     output = repr(obj)
    698     for idx,output_line in enumerate(output.splitlines()):
    699         if idx:

~/software_builds/modin/modin/pandas/dataframe.py in __repr__(self)
    168 
    169             num_cols += len(self.columns) - i
--> 170         result = repr(self._build_repr_df(num_rows, num_cols))
    171         if len(self.index) > num_rows or len(self.columns) > num_cols:
    172             # The split here is so that we don't repr pandas row lengths.

~/software_builds/modin/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    117         else:
    118             indexer = row_indexer
--> 119         return self.iloc[indexer]._query_compiler.to_pandas()
    120 
    121     def _update_inplace(self, new_query_compiler):

~/software_builds/modin/modin/backends/pandas/query_compiler.py in to_pandas(self)
    193 
    194     def to_pandas(self):
--> 195         return self._modin_frame.to_pandas()
    196 
    197     @classmethod

~/software_builds/modin/modin/engines/base/frame/data.py in to_pandas(self)
   1816             ErrorMessage.catch_bugs_and_request_email(
   1817                 not df.index.equals(self.index) or not df.columns.equals(self.columns),
-> 1818                 "Internal and external indices do not match.",
   1819             )
   1820             df.index = self.index

~/software_builds/modin/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"
---> 54                 " caused this error.\n{}".format(extra_log)
     55             )
     56 

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Internal and external indices do not match.
```



Discussion:
This issue has been mentioned on **Modin Discuss**. There might be relevant details there:

https://discuss.modin.org/t/modin-errors-out-on-pytz-timezone/119/19


Resolving patch:
diff --git a/modin/engines/base/frame/data.py b/modin/engines/base/frame/data.py
--- a/modin/engines/base/frame/data.py
+++ b/modin/engines/base/frame/data.py
@@ -1632,7 +1632,12 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
         """
         if isinstance(other, type(self)):
             other = [other]
-
+        if all(o.axes[axis].equals(self.axes[axis]) for o in other):
+            return (
+                self._partitions,
+                [self._simple_shuffle(axis, o) for o in other],
+                self.axes[axis].copy(),
+            )
         index_other_obj = [o.axes[axis] for o in other]
         joined_index = self._join_index_objects(axis, index_other_obj, how, sort)
         # We have to set these because otherwise when we perform the functions it may
@@ -1661,6 +1666,38 @@ def _copartition(self, axis, other, how, sort, force_repartition=False):
             reindexed_other_list.append(reindexed_other)
         return reindexed_self, reindexed_other_list, joined_index
 
+    def _simple_shuffle(self, axis, other):
+        """
+        Shuffle other rows or columns to match partitioning of self.
+
+        Notes
+        -----
+        This should only be called when `other.axes[axis]` and `self.axes[axis]`
+            are identical.
+
+        Parameters
+        ----------
+            axis: 0 or 1
+                The axis to shuffle over
+            other: BasePandasFrame
+                The BasePandasFrame to match to self object's partitioning
+
+        Returns
+        -------
+        BasePandasFrame
+            Shuffled version of `other`.
+        """
+        assert self.axes[axis].equals(
+            other.axes[axis]
+        ), "`_simple_shuffle` should only be used if axis is identical"
+        if axis == 0:
+            lengths = self._row_lengths
+        else:
+            lengths = self._column_widths
+        return other._frame_mgr_cls.simple_shuffle(
+            axis, other._partitions, lambda x: x, lengths
+        )
+
     def _binary_op(self, op, right_frame, join_type="outer"):
         """
         Perform an operation that requires joining with another dataframe.
diff --git a/modin/engines/base/frame/partition_manager.py b/modin/engines/base/frame/partition_manager.py
--- a/modin/engines/base/frame/partition_manager.py
+++ b/modin/engines/base/frame/partition_manager.py
@@ -328,6 +328,46 @@ def map_axis_partitions(
             right=None,
         )
 
+    @classmethod
+    def simple_shuffle(cls, axis, partitions, map_func, lengths):
+        """
+        Shuffle data using `lengths` via `map_func`
+
+        Parameters
+        ----------
+            axis : 0 or 1
+                The axis to perform the map across (0 - index, 1 - columns).
+            partitions : NumPy array
+                The partitions of Modin Frame.
+            map_func : callable
+                The function to apply.
+            lengths : list(int)
+                The list of lengths to shuffle the object
+
+        Returns
+        -------
+        NumPy array
+            An array of new partitions for a Modin Frame.
+        """
+        preprocessed_map_func = cls.preprocess_func(map_func)
+        partitions = cls.axis_partition(partitions, axis)
+        # For mapping across the entire axis, we don't maintain partitioning because we
+        # may want to line to partitioning up with another BlockPartitions object. Since
+        # we don't need to maintain the partitioning, this gives us the opportunity to
+        # load-balance the data as well.
+        result_blocks = np.array(
+            [
+                part.apply(
+                    preprocessed_map_func, _lengths=lengths, manual_partition=True
+                )
+                for part in partitions
+            ]
+        )
+        # If we are mapping over columns, they are returned to use the same as
+        # rows, so we need to transpose the returned 2D NumPy array to return
+        # the structure to the correct order.
+        return result_blocks.T if not axis else result_blocks
+
     @classmethod
     def concat(cls, axis, left_parts, right_parts):
         """Concatenate the blocks with another set of blocks.


Test patch:
diff --git a/modin/pandas/test/dataframe/test_indexing.py b/modin/pandas/test/dataframe/test_indexing.py
--- a/modin/pandas/test/dataframe/test_indexing.py
+++ b/modin/pandas/test/dataframe/test_indexing.py
@@ -1156,6 +1156,16 @@ def test___setitem__(data):
     df_equals(modin_df, pandas_df)
 
 
+def test___setitem__with_mismatched_partitions():
+    fname = "200kx99.csv"
+    np.savetxt(fname, np.random.randint(0, 100, size=(200_000, 99)), delimiter=",")
+    modin_df = pd.read_csv(fname)
+    pandas_df = pandas.read_csv(fname)
+    modin_df["new"] = pd.Series(list(range(len(modin_df))))
+    pandas_df["new"] = pandas.Series(list(range(len(pandas_df))))
+    df_equals(modin_df, pandas_df)
+
+
 def test___setitem__mask():
     # DataFrame mask:
     data = test_data["int_data"]