Repository: mars-project/mars

Issue:
[BUG] Filter on datetime64 failed with error `IndexError: tuple index out of range`
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Filter on datetime64 failed with error `IndexError: tuple index out of range`.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```python
In [9]: df = md.DataFrame({'a': [1, 2, 3], 'b': [1.1, 2.2, 3.3],
      ...:                                     'c': [datetime(2020, 1, 1), datetime.now(), datetime(2000, 3, 3, 11, 22, 23)]})
In[10]: df[(df['c'] > md.to_datetime('2020-08-01')) & (df['c'] < md.to_datetime('2020-11-01'))].head().execute()
Traceback (most recent call last):
  File "/Users/qinxuye/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3331, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-10-d9c9c3710287>", line 1, in <module>
    df[(df['c'] > md.to_datetime('2020-08-01')) & (df['c'] < md.to_datetime('2020-11-01'))].head().execute()
  File "/Users/qinxuye/Workspace/mars/mars/core.py", line 640, in execute
    return run()
  File "/Users/qinxuye/Workspace/mars/mars/core.py", line 636, in run
    self.data.execute(session, **kw)
  File "/Users/qinxuye/Workspace/mars/mars/core.py", line 376, in execute
    return run()
  File "/Users/qinxuye/Workspace/mars/mars/core.py", line 371, in run
    session.run(self, **kw)
  File "/Users/qinxuye/Workspace/mars/mars/session.py", line 500, in run
    result = self._sess.run(*tileables, **kw)
  File "/Users/qinxuye/Workspace/mars/mars/session.py", line 108, in run
    res = self._executor.execute_tileables(tileables, **kw)
  File "/Users/qinxuye/Workspace/mars/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/Users/qinxuye/Workspace/mars/mars/executor.py", line 861, in execute_tileables
    tileables, tileable_graph=tileable_graph)
  File "/Users/qinxuye/Workspace/mars/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 348, in build
    tileables, tileable_graph=tileable_graph)
  File "/Users/qinxuye/Workspace/mars/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 262, in build
    self._on_tile_failure(tileable_data.op, exc_info)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 301, in inner
    raise exc_info[1].with_traceback(exc_info[2]) from None
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 242, in build
    tiled = self._tile(tileable_data, tileable_graph)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 337, in _tile
    return super()._tile(tileable_data, tileable_graph)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 201, in _tile
    tds[0]._inplace_tile()
  File "/Users/qinxuye/Workspace/mars/mars/core.py", line 165, in _inplace_tile
    return handler.inplace_tile(self)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 136, in inplace_tile
    dispatched = self.dispatch(to_tile.op)
  File "/Users/qinxuye/Workspace/mars/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/Users/qinxuye/Workspace/mars/mars/tiles.py", line 119, in dispatch
    tiled = op_cls.tile(op)
  File "/Users/qinxuye/Workspace/mars/mars/dataframe/arithmetic/core.py", line 261, in tile
    return cls._tile_with_tensor(op)
  File "/Users/qinxuye/Workspace/mars/mars/dataframe/arithmetic/core.py", line 216, in _tile_with_tensor
    rechunk_size = other.nsplits[1] if op.axis == 'columns' or op.axis == 1 else other.nsplits[0]
IndexError: tuple index out of range
```


Discussion:

Resolving patch:
diff --git a/mars/dataframe/arithmetic/core.py b/mars/dataframe/arithmetic/core.py
--- a/mars/dataframe/arithmetic/core.py
+++ b/mars/dataframe/arithmetic/core.py
@@ -207,16 +207,23 @@ def _tile_scalar(cls, op):
 
     @classmethod
     def _tile_with_tensor(cls, op):
+        out = op.outputs[0]
+        axis = op.axis
+
         rhs_is_tensor = isinstance(op.rhs, TENSOR_TYPE)
         tensor, other = (op.rhs, op.lhs) if rhs_is_tensor else (op.lhs, op.rhs)
         if tensor.shape == other.shape:
             tensor = tensor.rechunk(other.nsplits)._inplace_tile()
         else:
             # shape differs only when dataframe add 1-d tensor, we need rechunk on columns axis.
-            rechunk_size = other.nsplits[1] if op.axis == 'columns' or op.axis == 1 else other.nsplits[0]
+            if op.axis in ['columns', 1] and other.ndim == 1:
+                # force axis == 0 if it's Series other than DataFrame
+                axis = 0
+            rechunk_size = other.nsplits[1] if axis == 'columns' or axis == 1 else other.nsplits[0]
             if tensor.ndim > 0:
                 tensor = tensor.rechunk((rechunk_size,))._inplace_tile()
 
+        cum_splits = [0] + np.cumsum(other.nsplits[axis]).tolist()
         out_chunks = []
         for out_index in itertools.product(*(map(range, other.chunk_shape))):
             tensor_chunk = tensor.cix[out_index[:tensor.ndim]]
@@ -224,19 +231,21 @@ def _tile_with_tensor(cls, op):
             out_op = op.copy().reset_key()
             inputs = [other_chunk, tensor_chunk] if rhs_is_tensor else [tensor_chunk, other_chunk]
             if isinstance(other_chunk, DATAFRAME_CHUNK_TYPE):
+                start = cum_splits[out_index[axis]]
+                end = cum_splits[out_index[axis] + 1]
+                chunk_dtypes = out.dtypes.iloc[start: end]
                 out_chunk = out_op.new_chunk(inputs, shape=other_chunk.shape, index=other_chunk.index,
-                                             dtypes=other_chunk.dtypes,
+                                             dtypes=chunk_dtypes,
                                              index_value=other_chunk.index_value,
                                              columns_value=other.columns_value)
             else:
                 out_chunk = out_op.new_chunk(inputs, shape=other_chunk.shape, index=other_chunk.index,
-                                             dtype=other_chunk.dtype,
+                                             dtype=out.dtype,
                                              index_value=other_chunk.index_value,
                                              name=other_chunk.name)
             out_chunks.append(out_chunk)
 
         new_op = op.copy()
-        out = op.outputs[0]
         if isinstance(other, SERIES_TYPE):
             return new_op.new_seriess(op.inputs, other.shape, nsplits=other.nsplits, dtype=out.dtype,
                                       index_value=other.index_value, chunks=out_chunks)
@@ -288,6 +297,8 @@ def execute(cls, ctx, op):
         if op.level is not None:
             # logical function like and may don't have `level` (for Series type)
             kw['level'] = op.level
+        if hasattr(other, 'ndim') and other.ndim == 0:
+            other = other.item()
         ctx[op.outputs[0].key] = getattr(df, func_name)(other, **kw)
 
     @classproperty
@@ -315,7 +326,10 @@ def _calc_properties(cls, x1, x2=None, axis='columns'):
                 and (x2 is None or pd.api.types.is_scalar(x2) or isinstance(x2, TENSOR_TYPE)):
             x2_dtype = x2.dtype if hasattr(x2, 'dtype') else type(x2)
             dtype = infer_dtype(x1.dtype, np.dtype(x2_dtype), cls._operator)
-            return {'shape': x1.shape, 'dtype': dtype, 'index_value': x1.index_value}
+            ret = {'shape': x1.shape, 'dtype': dtype, 'index_value': x1.index_value}
+            if pd.api.types.is_scalar(x2) or (hasattr(x2, 'ndim') and x2.ndim == 0):
+                ret['name'] = x1.name
+            return ret
 
         if isinstance(x1, (DATAFRAME_TYPE, DATAFRAME_CHUNK_TYPE)) and isinstance(
                 x2, (DATAFRAME_TYPE, DATAFRAME_CHUNK_TYPE)):
@@ -408,7 +422,10 @@ def _calc_properties(cls, x1, x2=None, axis='columns'):
                             (not np.isnan(x1.shape[0]) or not np.isnan(x2.shape[0])):
                         index_shape = x1.shape[0] if not np.isnan(x1.shape[0]) else x2.shape[0]
 
-            return {'shape': (index_shape,), 'dtype': dtype, 'index_value': index}
+            ret = {'shape': (index_shape,), 'dtype': dtype, 'index_value': index}
+            if x1.name == x2.name:
+                ret['name'] = x1.name
+            return ret
 
         raise NotImplementedError('Unknown combination of parameters')
 
diff --git a/mars/dataframe/merge/concat.py b/mars/dataframe/merge/concat.py
--- a/mars/dataframe/merge/concat.py
+++ b/mars/dataframe/merge/concat.py
@@ -202,7 +202,7 @@ def _base_concat(chunk, inputs):
                                 'and CategoricalChunk can be automatically concatenated')
 
         def _auto_concat_dataframe_chunks(chunk, inputs):
-            xdf = pd if isinstance(inputs[0], (pd.DataFrame, pd.Series)) else cudf
+            xdf = pd if isinstance(inputs[0], (pd.DataFrame, pd.Series)) or cudf is None else cudf
 
             if chunk.op.axis is not None:
                 return xdf.concat(inputs, axis=op.axis)
@@ -251,7 +251,7 @@ def _auto_concat_series_chunks(chunk, inputs):
                 if len(inputs) == 1:
                     concat = inputs[0]
                 else:
-                    xdf = pd if isinstance(inputs[0], pd.Series) else cudf
+                    xdf = pd if isinstance(inputs[0], pd.Series) or cudf is None else cudf
                     if chunk.op.axis is not None:
                         concat = xdf.concat(inputs, axis=chunk.op.axis)
                     else:
@@ -262,10 +262,10 @@ def _auto_concat_series_chunks(chunk, inputs):
 
         def _auto_concat_index_chunks(chunk, inputs):
             if len(inputs) == 1:
-                xdf = pd if isinstance(inputs[0], pd.Index) else cudf
+                xdf = pd if isinstance(inputs[0], pd.Index) or cudf is None else cudf
                 concat_df = xdf.DataFrame(index=inputs[0])
             else:
-                xdf = pd if isinstance(inputs[0], pd.Index) else cudf
+                xdf = pd if isinstance(inputs[0], pd.Index) or cudf is None else cudf
                 empty_dfs = [xdf.DataFrame(index=inp) for inp in inputs]
                 concat_df = xdf.concat(empty_dfs, axis=0)
             if getattr(chunk.index_value, 'should_be_monotonic', False):


Test patch:
diff --git a/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py b/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py
--- a/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py
+++ b/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py
@@ -22,6 +22,7 @@
 from mars.tests.core import TestBase, parameterized, ExecutorForTest
 from mars import tensor as mt
 from mars.tensor.datasource import array as from_array
+from mars.dataframe import to_datetime
 from mars.dataframe.datasource.dataframe import from_pandas
 from mars.dataframe.datasource.series import from_pandas as from_pandas_series
 from mars.dataframe.arithmetic.tests.test_arithmetic import comp_func
@@ -707,6 +708,21 @@ def testUfunc(self):
                 else:
                     pd.testing.assert_series_equal(result, expected)
 
+    def testDateTimeBin(self):
+        rs = np.random.RandomState(0)
+        df_raw = pd.DataFrame({'a': rs.randint(1000, size=10),
+                               'b': rs.rand(10),
+                               'c': [pd.Timestamp(rs.randint(1604000000, 1604481373))
+                                     for _ in range(10)]},
+                              index=pd.RangeIndex(9, -1, -1))
+        df = from_pandas(df_raw, chunk_size=5)
+        r = (df['c'] > to_datetime('2000-01-01')) & (df['c'] < to_datetime('2021-01-01'))
+
+        result = self.executor.execute_dataframe(r, concat=True)[0]
+        expected = (df_raw['c'] > pd.to_datetime('2000-01-01')) & \
+                   (df_raw['c'] < pd.to_datetime('2021-01-01'))
+        pd.testing.assert_series_equal(result, expected)
+
 
 if __name__ == '__main__':  # pragma: no cover
     unittest.main()