Repository: modin-project/modin

Issue:
DataFrame shallow copy acting deep
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: OS X 10.13.6
- **Modin installed from (source or binary)**: pip 
- **Modin version**: 0.4.0
- **Python version**:  3.7.2
- **Exact command to reproduce**: Run the below script

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
Modin currently ignores the deep parameter for `df.copy()` and always deep copies.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```df = pd.DataFrame({'X': list(range(5)), 'Y': list(range(5))})
df2 = df.copy(deep=False)
df['X'][0]=99
assert df['X'][0] == df2['X'][0], "Should be true for a shallow copy"```


Discussion:
I think that in order to fix this we just need QueryCompiler.copy() to take an option param `deep`. If deep is false then we shouldn't deep copy the constructor args. 
Hi @wuisawesome i try your idea, i add that option param and then in case it is False :

`return self.__constructor__(
        self.data, self.index, self.columns, self._dtype_cache
    )
`
instead of copy each of the constructor args, but it is still deep copying.
@wuisawesome, @ipacheco-uy The `__setitem__` issue is that the plasma store is immutable. We cannot update things inplace, and it is copying whenever we write some new data. We need to make sure that the references are propagated to the copies. We also enforce that `FrameManager` objects are immutable (and thus, `QueryCompiler` objects as well).

High level idea for how to implement this:

* All `DataFrame`s that are copies share a `QueryCompiler`. `df.copy()` will just return `DataFrame(query_compiler=self.query_compiler)`.
  * This will help with things like `df.index = [...]`, because no extra compute will be required. 
* When one `DataFrame` in the set has an `inplace` operation, we replace the `QueryCompiler` with the new one and propagate that new `QueryCompiler` to all of the other `DataFrame`s that depend on it. This should be fast because it is just updating the pointers.
@ipacheco-uy Is this an issue you are interested in trying to fix?

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -31,6 +31,16 @@ class BasePandasDataset(object):
         behavior of those objects and then use those objects to define the output type.
     """
 
+    # Siblings are other objects that share the same query compiler. We use this list
+    # to update inplace when there is a shallow copy.
+    _siblings = []
+
+    def _add_sibling(self, sibling):
+        sibling._siblings = self._siblings + [self]
+        self._siblings += [sibling]
+        for sib in self._siblings:
+            sib._siblings += [sibling]
+
     def _build_repr_df(self, num_rows, num_cols):
         # Add one here so that pandas automatically adds the dots
         # It turns out to be faster to extract 2 extra rows and columns than to
@@ -80,6 +90,8 @@ def _update_inplace(self, new_query_compiler):
         """
         old_query_compiler = self._query_compiler
         self._query_compiler = new_query_compiler
+        for sib in self._siblings:
+            sib._query_compiler = new_query_compiler
         old_query_compiler.free()
 
     def _validate_other(
@@ -652,7 +664,11 @@ def copy(self, deep=True):
         Returns:
             A new DataFrame pointing to the same partitions as this one.
         """
-        return self.__constructor__(query_compiler=self._query_compiler.copy())
+        if deep:
+            return self.__constructor__(query_compiler=self._query_compiler.copy())
+        new_obj = self.__constructor__(query_compiler=self._query_compiler)
+        self._add_sibling(new_obj)
+        return new_obj
 
     def count(self, axis=0, level=None, numeric_only=False):
         """Get the count of non-null objects in the DataFrame.
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -58,11 +58,16 @@ def __init__(
             query_compiler: A query compiler object to manage distributed computation.
         """
         if isinstance(data, (DataFrame, Series)):
-            self._query_compiler = data._query_compiler.copy()
+            self._query_compiler = data._query_compiler
             if isinstance(data, Series) and data.name is None:
                 self.columns = [0]
+            else:
+                data._add_sibling(self)
         # Check type of data and use appropriate constructor
         elif query_compiler is None:
+            warnings.warn(
+                "Distributing {} object. This may take some time.".format(type(data))
+            )
             if is_list_like(data) and not is_dict_like(data):
                 data = [
                     obj._to_pandas() if isinstance(obj, Series) else obj for obj in data
@@ -74,9 +79,6 @@ def __init__(
                     k: v._to_pandas() if isinstance(v, Series) else v
                     for k, v in data.items()
                 }
-            warnings.warn(
-                "Distributing {} object. This may take some time.".format(type(data))
-            )
             pandas_df = pandas.DataFrame(
                 data=data, index=index, columns=columns, dtype=dtype, copy=copy
             )
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -321,7 +321,7 @@ def append(self, to_append, ignore_index=False, verify_integrity=False):
                     if isinstance(to_append[i], Series):
                         to_append[i].name = 0
                         to_append[i] = DataFrame(to_append[i])
-                return DataFrame(self).append(
+                return DataFrame(self.copy()).append(
                     to_append,
                     ignore_index=ignore_index,
                     verify_integrity=verify_integrity,
@@ -332,7 +332,7 @@ def append(self, to_append, ignore_index=False, verify_integrity=False):
             to_append = [to_append._query_compiler]
         elif isinstance(to_append, DataFrame):
             self.name = 0
-            return DataFrame(self).append(
+            return DataFrame(self.copy()).append(
                 to_append, ignore_index=ignore_index, verify_integrity=verify_integrity
             )
         else:
@@ -855,7 +855,7 @@ def rename(self, index=None, **kwargs):
         else:
             from .dataframe import DataFrame
 
-            result = DataFrame(self).rename(index=index, **kwargs).squeeze()
+            result = DataFrame(self.copy()).rename(index=index, **kwargs).squeeze()
             result.name = self.name
             return result
 
@@ -886,7 +886,9 @@ def reset_index(self, level=None, drop=False, name=None, inplace=False):
                 obj.name = name
             from .dataframe import DataFrame
 
-            return DataFrame(self).reset_index(level=level, drop=drop, inplace=inplace)
+            return DataFrame(self.copy()).reset_index(
+                level=level, drop=drop, inplace=inplace
+            )
 
     def rdivmod(self, other, level=None, fill_value=None, axis=0):
         return self._default_to_pandas(
@@ -947,7 +949,7 @@ def sort_values(
         # is None, so we do this to avoid a KeyError.
         by = self.name if self.name is not None else 0
         result = (
-            DataFrame(self)
+            DataFrame(self.copy())
             .sort_values(
                 by=by,
                 ascending=ascending,
@@ -1018,7 +1020,7 @@ def to_frame(self, name=None):
         self_cp = self.copy()
         if name is not None:
             self_cp.name = name
-        return DataFrame(self)
+        return DataFrame(self.copy())
 
     def to_list(self):
         return self._default_to_pandas(pandas.Series.to_list)


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -631,6 +631,13 @@ def test_copy(self, data):
         assert new_modin_df is not modin_df
         df_equals(new_modin_df, modin_df)
 
+        # Shallow copy tests
+        modin_df = pd.DataFrame(data)
+        modin_df_cp = modin_df.copy(False)
+
+        modin_df[modin_df.columns[0]] = 0
+        df_equals(modin_df, modin_df_cp)
+
     @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
     def test_dtypes(self, data):
         modin_df = pd.DataFrame(data)