Repository: modin-project/modin

Issue:
Error when setting new values for a column

### Describe the problem
<!-- Describe the problem clearly here. -->
Our internal index tracking is incorrectly trigger computation on some partitions without supplying it with the values to compute on.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```python
df['col1'] = np.log(df['col1'])
```

Full Error:
```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
~/software_builds/modin/modin/backends/pandas/query_compiler.py in setitem(df, internal_indices)
   2040             try:
-> 2041                 _setitem()
   2042             except ValueError:

~/software_builds/modin/modin/backends/pandas/query_compiler.py in _setitem()
   2035                     if axis == 0:
-> 2036                         df[df.columns[internal_indices]] = value
   2037                     else:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py in __setitem__(self, key, value)
   3366         elif isinstance(key, (Series, np.ndarray, list, Index)):
-> 3367             self._setitem_array(key, value)
   3368         else:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py in _setitem_array(self, key, value)
   3394                 self._check_setitem_copy()
-> 3395                 self.loc._setitem_with_indexer((slice(None), indexer), value)
   3396 

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/indexing.py in _setitem_with_indexer(self, indexer, value)
    610                     if len(labels) != len(value):
--> 611                         raise ValueError('Must have equal len keys and value '
    612                                          'when setting with an iterable')

ValueError: Must have equal len keys and value when setting with an iterable

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-2-43a2a4c0ee3d> in <module>
----> 1 df_train['SalePrice'] = np.log(df_train['SalePrice'])

~/software_builds/modin/modin/pandas/dataframe.py in __setitem__(self, key, value)
   1955             self._update_inplace(new_self._query_compiler)
   1956         else:
-> 1957             self._update_inplace(self._query_compiler.setitem(0, key, value))
   1958 
   1959     def __unicode__(self):

~/software_builds/modin/modin/backends/pandas/query_compiler.py in setitem(self, axis, key, value)
   2053         if is_list_like(value):
   2054             new_data = self.data.apply_func_to_select_indices_along_full_axis(
-> 2055                 axis, prepared_func, numeric_indices, keep_remaining=True
   2056             )
   2057         else:

~/software_builds/modin/modin/engines/base/frame/partition_manager.py in apply_func_to_select_indices_along_full_axis(self, axis, func, indices, keep_remaining)
    951                             preprocessed_func, internal_indices=partitions_dict[i]
    952                         )
--> 953                         for i in range(len(partitions_for_remaining))
    954                     ]
    955                 )

~/software_builds/modin/modin/engines/base/frame/partition_manager.py in <listcomp>(.0)
    951                             preprocessed_func, internal_indices=partitions_dict[i]
    952                         )
--> 953                         for i in range(len(partitions_for_remaining))
    954                     ]
    955                 )

~/software_builds/modin/modin/engines/base/frame/axis_partition.py in apply(self, func, num_splits, other_axis_partition, maintain_partitioning, **kwargs)
    139         args = [self.axis, func, num_splits, kwargs, maintain_partitioning]
    140         args.extend(self.list_of_blocks)
--> 141         return self._wrap_partitions(self.deploy_axis_func(*args))
    142 
    143     def shuffle(self, func, lengths, **kwargs):

~/software_builds/modin/modin/engines/base/frame/axis_partition.py in deploy_axis_func(cls, axis, func, num_splits, kwargs, maintain_partitioning, *partitions)
    185 
    186         dataframe = pandas.concat(partitions, axis=axis, copy=False)
--> 187         result = func(dataframe, **kwargs)
    188         if isinstance(result, pandas.Series):
    189             if num_splits == 1:

~/software_builds/modin/modin/backends/pandas/query_compiler.py in helper(df, internal_indices)
    158                 if len(internal_indices) > 0:
    159                     return pandas_func(df, internal_indices=internal_indices, **kwargs)
--> 160                 return pandas_func(df, **kwargs)
    161 
    162         return helper

~/software_builds/modin/modin/backends/pandas/query_compiler.py in setitem(df, internal_indices)
   2043                 # TODO: This is a workaround for a pyarrow serialization issue
   2044                 df = df.copy()
-> 2045                 _setitem()
   2046             return df
   2047 

~/software_builds/modin/modin/backends/pandas/query_compiler.py in _setitem()
   2034                 else:
   2035                     if axis == 0:
-> 2036                         df[df.columns[internal_indices]] = value
   2037                     else:
   2038                         df.iloc[internal_indices] = value

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py in __setitem__(self, key, value)
   3365             self._setitem_frame(key, value)
   3366         elif isinstance(key, (Series, np.ndarray, list, Index)):
-> 3367             self._setitem_array(key, value)
   3368         else:
   3369             # set column

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py in _setitem_array(self, key, value)
   3393                 indexer = self.loc._convert_to_indexer(key, axis=1)
   3394                 self._check_setitem_copy()
-> 3395                 self.loc._setitem_with_indexer((slice(None), indexer), value)
   3396 
   3397     def _setitem_frame(self, key, value):

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/indexing.py in _setitem_with_indexer(self, indexer, value)
    609 
    610                     if len(labels) != len(value):
--> 611                         raise ValueError('Must have equal len keys and value '
    612                                          'when setting with an iterable')
    613 

ValueError: Must have equal len keys and value when setting with an iterable
```


Discussion:

Resolving patch:
diff --git a/modin/engines/base/frame/partition_manager.py b/modin/engines/base/frame/partition_manager.py
--- a/modin/engines/base/frame/partition_manager.py
+++ b/modin/engines/base/frame/partition_manager.py
@@ -685,10 +685,17 @@ def internal(block_idx, global_index):
             count_for_each_partition = np.array(
                 [(partition_ids == i).sum() for i in range(len(cumulative))]
             ).cumsum()
-            # compute the internal indices and pair those with the partition index.
-            partition_ids_with_indices = [
-                (0, internal(0, indices[slice(count_for_each_partition[0])]))
-            ] + [
+            # Compute the internal indices and pair those with the partition index.
+            # If the first partition has any values we need to return, compute those
+            # first to make the list comprehension easier. Otherwise, just append the
+            # rest of the values to an empty list.
+            if count_for_each_partition[0] > 0:
+                first_partition_indices = [
+                    (0, internal(0, indices[slice(count_for_each_partition[0])]))
+                ]
+            else:
+                first_partition_indices = []
+            partition_ids_with_indices = first_partition_indices + [
                 (
                     i,
                     internal(


Test patch: