Repository: modin-project/modin

Issue:
Dataframe shift breaks with Modin distribution but not stock pandas
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`): 0.10.1
- **Python version**: 3.7.9
- **Code we can use to reproduce**:
-  contact alyssa.unell@intel.com for information regarding the code and dataset


### Describe the problem
Shifting of a dataframe using modin distribution breaks

### Source code / logs
UserWarning: Distributing <class 'NoneType'> object. This may take some time.
Traceback (most recent call last):
  File "deeplearning/RecSys2020/02_ModelsCompetition/XGBoost2/sub040_1_save_public.py", line 731, in <module>
    add_timeshift(train, test, shift=1)
  File "deeplearning/RecSys2020/02_ModelsCompetition/XGBoost2/sub040_1_save_public.py", line 339, in add_timeshift
    gf["b_user_id_shifted"] = gf["b_user_id"].shift(periods=shift)
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/pandas/series.py", line 1361, in shift
    periods=periods, freq=freq, axis=axis, fill_value=fill_value
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/pandas/base.py", line 2319, in shift
    new_frame.index = self.index.copy()
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/pandas/base.py", line 562, in _set_index
    self._query_compiler.index = new_index
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/backends/pandas/query_compiler.py", line 91, in set_axis
    self._modin_frame.index = idx
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/engines/base/frame/data.py", line 250, in _set_index
    new_index = self._validate_set_axis(new_index, self._index_cache)
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/engines/base/frame/data.py", line 212, in _validate_set_axis
    "new values have %d elements" % (old_len, new_len)
ValueError: Length mismatch: Expected axis has 199999 elements, new values have 200000 elements


Dataframe shift breaks with Modin distribution but not stock pandas
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`): 0.10.1
- **Python version**: 3.7.9
- **Code we can use to reproduce**:
-  contact alyssa.unell@intel.com for information regarding the code and dataset


### Describe the problem
Shifting of a dataframe using modin distribution breaks

### Source code / logs
UserWarning: Distributing <class 'NoneType'> object. This may take some time.
Traceback (most recent call last):
  File "deeplearning/RecSys2020/02_ModelsCompetition/XGBoost2/sub040_1_save_public.py", line 731, in <module>
    add_timeshift(train, test, shift=1)
  File "deeplearning/RecSys2020/02_ModelsCompetition/XGBoost2/sub040_1_save_public.py", line 339, in add_timeshift
    gf["b_user_id_shifted"] = gf["b_user_id"].shift(periods=shift)
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/pandas/series.py", line 1361, in shift
    periods=periods, freq=freq, axis=axis, fill_value=fill_value
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/pandas/base.py", line 2319, in shift
    new_frame.index = self.index.copy()
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/pandas/base.py", line 562, in _set_index
    self._query_compiler.index = new_index
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/backends/pandas/query_compiler.py", line 91, in set_axis
    self._modin_frame.index = idx
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/engines/base/frame/data.py", line 250, in _set_index
    new_index = self._validate_set_axis(new_index, self._index_cache)
  File "/localdisk/alyssa/anaconda3/envs/aikit-modin/lib/python3.7/site-packages/modin/engines/base/frame/data.py", line 212, in _validate_set_axis
    "new values have %d elements" % (old_len, new_len)
ValueError: Length mismatch: Expected axis has 199999 elements, new values have 200000 elements




Discussion:

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -2336,20 +2336,15 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=no_default):
 
         if freq is None:
             if axis == "index" or axis == 0:
-                if periods > 0:
-                    dropped_df = self.drop(self.index[-periods:])
-                    new_frame = filled_df.append(dropped_df, ignore_index=True)
-                    new_frame.index = self.index.copy()
-                    if isinstance(self, DataFrame):
-                        new_frame.columns = self.columns.copy()
-                    return new_frame
-                else:
-                    dropped_df = self.drop(self.index[:-periods])
-                    new_frame = dropped_df.append(filled_df, ignore_index=True)
-                    new_frame.index = self.index.copy()
-                    if isinstance(self, DataFrame):
-                        new_frame.columns = self.columns.copy()
-                    return new_frame
+                new_frame = (
+                    filled_df.append(self.iloc[:-periods], ignore_index=True)
+                    if periods > 0
+                    else self.iloc[-periods:].append(filled_df, ignore_index=True)
+                )
+                new_frame.index = self.index.copy()
+                if isinstance(self, DataFrame):
+                    new_frame.columns = self.columns.copy()
+                return new_frame
             else:
                 if not isinstance(self, DataFrame):
                     raise ValueError(


Test patch:
diff --git a/modin/pandas/test/dataframe/test_default.py b/modin/pandas/test/dataframe/test_default.py
--- a/modin/pandas/test/dataframe/test_default.py
+++ b/modin/pandas/test/dataframe/test_default.py
@@ -764,18 +764,16 @@ def test_resample_specific(rule, closed, label, on, level):
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
-@pytest.mark.parametrize("index", ["default", "ndarray"])
+@pytest.mark.parametrize("index", ["default", "ndarray", "has_duplicates"])
 @pytest.mark.parametrize("axis", [0, 1])
 @pytest.mark.parametrize("periods", [0, 1, -1, 10, -10, 1000000000, -1000000000])
 def test_shift(data, index, axis, periods):
-    if index == "default":
-        modin_df = pd.DataFrame(data)
-        pandas_df = pandas.DataFrame(data)
-    elif index == "ndarray":
+    modin_df, pandas_df = create_test_dfs(data)
+    if index == "ndarray":
         data_column_length = len(data[next(iter(data))])
-        index_data = np.arange(2, data_column_length + 2)
-        modin_df = pd.DataFrame(data, index=index_data)
-        pandas_df = pandas.DataFrame(data, index=index_data)
+        modin_df.index = pandas_df.index = np.arange(2, data_column_length + 2)
+    elif index == "has_duplicates":
+        modin_df.index = pandas_df.index = list(modin_df.index[:-3]) + [0, 1, 2]
 
     df_equals(
         modin_df.shift(periods=periods, axis=axis),