Repository: modin-project/modin

Issue:
Error when trying to build a dataframe column by column
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: MacOS 10.14
- **Modin installed from (source or binary)**: source
- **Modin version**: Current master
- **Python version**: 3.6
- **Exact command to reproduce**:
```
import modin.pandas as pd
import numpy as np

cols = ['a', 'b', 'c']
df = pd.DataFrame(columns=cols)
for col in cols:
    df[col] = np.arange(10)
```

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
When trying to build a dataframe column-by-column, I get an `AssertionError`

### Source code / logs
Traceback:
```
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-5-7f2184cfd6d7> in <module>
      2 df = pd.DataFrame(columns=cols)
      3 for col in cols:
----> 4     df[col] = np.arange(10)

~/Documents/modin/modin/pandas/dataframe.py in __setitem__(self, key, value)
   4605         else:
   4606             loc = self.columns.get_loc(key)
-> 4607             self.__delitem__(key)
   4608             self.insert(loc=loc, column=key, value=value)
   4609 

~/Documents/modin/modin/pandas/dataframe.py in __delitem__(self, key)
   4690         if key not in self:
   4691             raise KeyError(key)
-> 4692         self._update_inplace(new_query_compiler=self._query_compiler.delitem(key))
   4693 
   4694     def __finalize__(self, other, method=None, **kwargs):

~/Documents/modin/modin/data_management/query_compiler/pandas_query_compiler.py in delitem(self, key)
   2294     # These will change the shape of the resulting data.
   2295     def delitem(self, key):
-> 2296         return self.drop(columns=[key])
   2297 
   2298     def drop(self, index=None, columns=None):

~/Documents/modin/modin/data_management/query_compiler/pandas_query_compiler.py in drop(self, index, columns)
   2336             new_columns = self.columns[~self.columns.isin(columns)]
   2337             new_dtypes = self.dtypes.drop(columns)
-> 2338         return self.__constructor__(new_data, new_index, new_columns, new_dtypes)
   2339 
   2340     # END __delitem__ and drop

~/Documents/modin/modin/data_management/query_compiler/pandas_query_compiler.py in __constructor__(self, block_paritions_object, index, columns, dtypes)
     41     def __constructor__(self, block_paritions_object, index, columns, dtypes=None):
     42         """By default, constructor method will invoke an init"""
---> 43         return type(self)(block_paritions_object, index, columns, dtypes)
     44 
     45     # Index, columns and dtypes objects

~/Documents/modin/modin/data_management/query_compiler/pandas_query_compiler.py in __init__(self, block_partitions_object, index, columns, dtypes)
     32         dtypes=None,
     33     ):
---> 34         assert isinstance(block_partitions_object, BaseBlockPartitions)
     35         self.data = block_partitions_object
     36         self.index = index

AssertionError: 
```


Discussion:

Resolving patch:
diff --git a/modin/data_management/query_compiler/pandas_query_compiler.py b/modin/data_management/query_compiler/pandas_query_compiler.py
--- a/modin/data_management/query_compiler/pandas_query_compiler.py
+++ b/modin/data_management/query_compiler/pandas_query_compiler.py
@@ -159,11 +159,17 @@ def _prepare_method(self, pandas_func, **kwargs):
         if self._is_transposed:
 
             def helper(df, internal_indices=[]):
+                if len(internal_indices) > 0:
+                    return pandas_func(
+                        df.T, internal_indices=internal_indices, **kwargs
+                    )
                 return pandas_func(df.T, **kwargs)
 
         else:
 
             def helper(df, internal_indices=[]):
+                if len(internal_indices) > 0:
+                    return pandas_func(df, internal_indices=internal_indices, **kwargs)
                 return pandas_func(df, **kwargs)
 
         return helper
@@ -1724,11 +1730,8 @@ def describe(self, **kwargs):
                 # Describe all columns
                 new_columns = self.columns
 
-        def describe_builder(df, **kwargs):
-            try:
-                return pandas.DataFrame.describe(df, **kwargs)
-            except ValueError:
-                return pandas.DataFrame(index=df.index)
+        def describe_builder(df, internal_indices=[], **kwargs):
+            return df.iloc[:, internal_indices].describe(**kwargs)
 
         # Apply describe and update indices, columns, and dtypes
         func = self._prepare_method(describe_builder, **kwargs)
@@ -2304,6 +2307,37 @@ def getitem(df, internal_indices=[]):
         new_index = self.index[key]
         return self.__constructor__(result, new_index, self.columns, self._dtype_cache)
 
+    def setitem(self, key, value):
+        """Set the column defined by `key` to the `value` provided.
+
+        Args:
+            key: The column name to set.
+            value: The value to set the column to.
+
+        Returns:
+             A new PandasDataManager
+        """
+
+        def setitem(df, internal_indices=[]):
+            if len(internal_indices) == 1:
+                df[df.columns[internal_indices[0]]] = value
+            else:
+                df[df.columns[internal_indices]] = value
+            return df
+
+        numeric_indices = list(self.columns.get_indexer_for([key]))
+        prepared_func = self._prepare_method(setitem)
+        if is_list_like(value):
+            value = list(value)
+            new_data = self.data.apply_func_to_select_indices_along_full_axis(
+                0, prepared_func, numeric_indices, keep_remaining=True
+            )
+        else:
+            new_data = self.data.apply_func_to_select_indices(
+                0, prepared_func, numeric_indices, keep_remaining=True
+            )
+        return self.__constructor__(new_data, self.index, self.columns)
+
     # END __getitem__ methods
 
     # Drop
diff --git a/modin/engines/base/block_partitions.py b/modin/engines/base/block_partitions.py
--- a/modin/engines/base/block_partitions.py
+++ b/modin/engines/base/block_partitions.py
@@ -828,7 +828,7 @@ def apply_func_to_select_indices_along_full_axis(
             A new BaseBlockPartitions object, the type of object that called this.
         """
         if self.partitions.size == 0:
-            return np.array([[]])
+            return self.__constructor__(np.array([[]]))
         if isinstance(indices, dict):
             dict_indices = indices
             indices = list(indices.keys())
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -4723,13 +4723,21 @@ def __setattr__(self, key, value):
 
     def __setitem__(self, key, value):
         if not isinstance(key, str):
-            return self._default_to_pandas(pandas.DataFrame.__setitem__, key, value)
+
+            def setitem_without_string_columns(df):
+                df[key] = value
+                return df
+
+            return self._update_inplace(
+                self._default_to_pandas(setitem_without_string_columns)._query_compiler
+            )
         if key not in self.columns:
             self.insert(loc=len(self.columns), column=key, value=value)
+        elif len(self.index) == 0:
+            new_self = DataFrame({key: value}, columns=self.columns)
+            self._update_inplace(new_self._query_compiler)
         else:
-            loc = self.columns.get_loc(key)
-            self.__delitem__(key)
-            self.insert(loc=loc, column=key, value=value)
+            self._update_inplace(self._query_compiler.setitem(key, value))
 
     def __len__(self):
         """Gets the length of the DataFrame.


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -4311,14 +4311,55 @@ def test___getattr__(request, data):
         assert isinstance(df2.columns, pandas.Index)
 
 
-@pytest.mark.skip(reason="Defaulting to Pandas")
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test___setitem__(data):
     modin_df = pd.DataFrame(data)
-    pandas_df = pandas.DataFrame(data)  # noqa F841
+    pandas_df = pandas.DataFrame(data)
 
-    with pytest.raises(NotImplementedError):
-        modin_df.__setitem__(None, None)
+    modin_df.__setitem__(modin_df.columns[-1], 1)
+    pandas_df.__setitem__(pandas_df.columns[-1], 1)
+    df_equals(modin_df, pandas_df)
+
+    modin_df = pd.DataFrame(data)
+    pandas_df = pandas.DataFrame(data)
+
+    modin_df[modin_df.columns[0]] = np.arange(len(modin_df))
+    pandas_df[pandas_df.columns[0]] = np.arange(len(pandas_df))
+    df_equals(modin_df, pandas_df)
+
+    modin_df = pd.DataFrame(columns=modin_df.columns)
+    pandas_df = pandas.DataFrame(columns=pandas_df.columns)
+
+    for col in modin_df.columns:
+        modin_df[col] = np.arange(1000)
+
+    for col in pandas_df.columns:
+        pandas_df[col] = np.arange(1000)
+
+    df_equals(modin_df, pandas_df)
+
+    # Transpose test
+    modin_df = pd.DataFrame(data).T
+    pandas_df = pandas.DataFrame(data).T
+
+    # We default to pandas on non-string column names
+    if not all(isinstance(c, str) for c in modin_df.columns):
+        with pytest.warns(UserWarning):
+            modin_df[modin_df.columns[0]] = 0
+    else:
+        modin_df[modin_df.columns[0]] = 0
+
+    pandas_df[pandas_df.columns[0]] = 0
+
+    df_equals(modin_df, pandas_df)
+
+    modin_df.columns = [str(i) for i in modin_df.columns]
+    pandas_df.columns = [str(i) for i in pandas_df.columns]
+
+    modin_df[modin_df.columns[0]] = 0
+    pandas_df[pandas_df.columns[0]] = 0
+
+    df_equals(modin_df, pandas_df)
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)