Repository: mars-project/mars

Issue:
[BUG] `df.reset_index(drop=True)` run failed when df's shape is unknown
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
`df.reset_index(drop=True)` run failed when df's shape is unknown.

**To Reproduce**

``` Python
In [25]: df = md.DataFrame(mt.random.rand(10, 3), chunk_size=3)                                     

In [26]: df.sort_values(0).reset_index(drop=True).execute()                                         
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-26-e0c111d55eb4> in <module>
----> 1 df.sort_values(0).reset_index(drop=True).execute()

~/Documents/mars_dev/mars/mars/core.py in execute(self, session, **kw)
    559 
    560     def execute(self, session=None, **kw):
--> 561         self._data.execute(session, **kw)
    562         return self
    563 

~/Documents/mars_dev/mars/mars/core.py in execute(self, session, **kw)
    368 
    369         # no more fetch, thus just fire run
--> 370         session.run(self, **kw)
    371         # return Tileable or ExecutableTuple itself
    372         return self

~/Documents/mars_dev/mars/mars/session.py in run(self, *tileables, **kw)
    430         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
    431                           for t in tileables)
--> 432         result = self._sess.run(*tileables, **kw)
    433 
    434         for t in tileables:

~/Documents/mars_dev/mars/mars/session.py in run(self, *tileables, **kw)
     98             # set number of running cores
     99             self.context.set_ncores(kw['n_parallel'])
--> 100             res = self._executor.execute_tileables(tileables, **kw)
    101             return res
    102 

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Documents/mars_dev/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Documents/mars_dev/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose)
    817                 # build chunk graph, tile will be done during building
    818                 chunk_graph = chunk_graph_builder.build(
--> 819                     tileables, tileable_graph=tileable_graph)
    820                 tileable_graph = chunk_graph_builder.prev_tileable_graph
    821                 temp_result_keys = set(result_keys)

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Documents/mars_dev/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Documents/mars_dev/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    340 
    341         chunk_graph = super().build(
--> 342             tileables, tileable_graph=tileable_graph)
    343         self._iterative_chunk_graphs.append(chunk_graph)
    344         if len(self._interrupted_ops) == 0:

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Documents/mars_dev/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Documents/mars_dev/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    253                     # for further execution
    254                     partial_tiled_chunks = \
--> 255                         self._on_tile_failure(tileable_data.op, exc_info)
    256                     if partial_tiled_chunks is not None and \
    257                             len(partial_tiled_chunks) > 0:

~/Documents/mars_dev/mars/mars/tiles.py in inner(op, exc_info)
    292                     on_tile_failure(op, exc_info)
    293                 else:
--> 294                     raise exc_info[1].with_traceback(exc_info[2]) from None
    295         return inner
    296 

~/Documents/mars_dev/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    233                 continue
    234             try:
--> 235                 tiled = self._tile(tileable_data, tileable_graph)
    236                 tiled_op.add(tileable_data.op)
    237                 for t, td in zip(tileable_data.op.outputs, tiled):

~/Documents/mars_dev/mars/mars/tiles.py in _tile(self, tileable_data, tileable_graph)
    328         if any(inp.op in self._interrupted_ops for inp in tileable_data.inputs):
    329             raise TilesError('Tile fail due to failure of inputs')
--> 330         return super()._tile(tileable_data, tileable_graph)
    331 
    332     @kernel_mode

~/Documents/mars_dev/mars/mars/tiles.py in _tile(self, tileable_data, tileable_graph)
    191                 t._nsplits = o.nsplits
    192         elif on_tile is None:
--> 193             tds[0]._inplace_tile()
    194         else:
    195             tds = on_tile(tileable_data.op.outputs, tds)

~/Documents/mars_dev/mars/mars/core.py in _inplace_tile(self)
    160 
    161     def _inplace_tile(self):
--> 162         return handler.inplace_tile(self)
    163 
    164     def __getattr__(self, attr):

~/Documents/mars_dev/mars/mars/tiles.py in inplace_tile(self, to_tile)
    126         if not to_tile.is_coarse():
    127             return to_tile
--> 128         dispatched = self.dispatch(to_tile.op)
    129         self._assign_to([d.data for d in dispatched], to_tile.op.outputs)
    130         return to_tile

~/Documents/mars_dev/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Documents/mars_dev/mars/mars/tiles.py in dispatch(self, op)
    113             return self._handlers[op_cls](op)
    114         try:
--> 115             return op_cls.tile(op)
    116         except NotImplementedError as ex:
    117             cause = ex

~/Documents/mars_dev/mars/mars/dataframe/base/reset_index.py in tile(cls, op)
    130     def tile(cls, op):
    131         if isinstance(op.inputs[0], DATAFRAME_TYPE):
--> 132             return cls._tile_dataframe(op)
    133         else:
    134             return cls._tile_series(op)

~/Documents/mars_dev/mars/mars/dataframe/base/reset_index.py in _tile_dataframe(cls, op)
    101         for c in in_df.chunks:
    102             if is_range_index:
--> 103                 index_value = parse_index(pd.RangeIndex(cum_range[c.index[0]], cum_range[c.index[0] + 1]))
    104             else:
    105                 index_value = out_df.index_value

~/miniconda3/lib/python3.7/site-packages/pandas/core/indexes/range.py in __new__(cls, start, stop, step, dtype, copy, name)
    102             start, stop = 0, start
    103         else:
--> 104             stop = ensure_python_int(stop)
    105 
    106         step = ensure_python_int(step) if step is not None else 1

~/miniconda3/lib/python3.7/site-packages/pandas/core/dtypes/common.py in ensure_python_int(value)
    200         assert new_value == value
    201     except (TypeError, ValueError, AssertionError):
--> 202         raise TypeError(msg.format(type(value), value))
    203     return new_value
    204 

TypeError: Wrong type <class 'numpy.float64'> for value nan
```


Discussion:

Resolving patch:
diff --git a/mars/dataframe/arithmetic/core.py b/mars/dataframe/arithmetic/core.py
--- a/mars/dataframe/arithmetic/core.py
+++ b/mars/dataframe/arithmetic/core.py
@@ -70,7 +70,7 @@ def _set_inputs(self, inputs):
         else:
             if isinstance(self._lhs, (DATAFRAME_TYPE, SERIES_TYPE)):
                 self._lhs = self._inputs[0]
-            elif np.isscalar(self._lhs):
+            elif pd.api.types.is_scalar(self._lhs):
                 self._rhs = self._inputs[0]
 
 
@@ -181,7 +181,7 @@ def _tile_series_dataframe(cls, op):
 
     @classmethod
     def _tile_scalar(cls, op):
-        tileable = op.rhs if np.isscalar(op.lhs) else op.lhs
+        tileable = op.rhs if pd.api.types.is_scalar(op.lhs) else op.lhs
         df = op.outputs[0]
         out_chunks = []
         for chunk in tileable.chunks:
@@ -270,7 +270,7 @@ def execute(cls, ctx, op):
                 func_name = getattr(cls, '_rfunc_name')
             else:
                 func_name = getattr(cls, '_func_name')
-        elif np.isscalar(op.lhs) or isinstance(op.lhs, np.ndarray):
+        elif pd.api.types.is_scalar(op.lhs) or isinstance(op.lhs, np.ndarray):
             df = ctx[op.rhs.key]
             other = op.lhs
             func_name = getattr(cls, '_rfunc_name')
@@ -297,10 +297,10 @@ def _operator(self):
     @classmethod
     def _calc_properties(cls, x1, x2=None, axis='columns'):
         if isinstance(x1, (DATAFRAME_TYPE, DATAFRAME_CHUNK_TYPE)) \
-                and (x2 is None or np.isscalar(x2) or isinstance(x2, TENSOR_TYPE)):
+                and (x2 is None or pd.api.types.is_scalar(x2) or isinstance(x2, TENSOR_TYPE)):
             if x2 is None:
                 dtypes = x1.dtypes
-            elif np.isscalar(x2):
+            elif pd.api.types.is_scalar(x2):
                 dtypes = infer_dtypes(x1.dtypes, pd.Series(np.array(x2).dtype), cls._operator)
             elif x1.dtypes is not None and isinstance(x2, TENSOR_TYPE):
                 dtypes = pd.Series(
@@ -312,7 +312,7 @@ def _calc_properties(cls, x1, x2=None, axis='columns'):
                     'columns_value': x1.columns_value, 'index_value': x1.index_value}
 
         if isinstance(x1, (SERIES_TYPE, SERIES_CHUNK_TYPE)) \
-                and (x2 is None or np.isscalar(x2) or isinstance(x2, TENSOR_TYPE)):
+                and (x2 is None or pd.api.types.is_scalar(x2) or isinstance(x2, TENSOR_TYPE)):
             x2_dtype = x2.dtype if hasattr(x2, 'dtype') else type(x2)
             dtype = infer_dtype(x1.dtype, np.dtype(x2_dtype), cls._operator)
             return {'shape': x1.shape, 'dtype': dtype, 'index_value': x1.index_value}
@@ -436,7 +436,7 @@ def _new_chunks(self, inputs, kws=None, **kw):
 
     @staticmethod
     def _process_input(x):
-        if isinstance(x, (DATAFRAME_TYPE, SERIES_TYPE)) or np.isscalar(x):
+        if isinstance(x, (DATAFRAME_TYPE, SERIES_TYPE)) or pd.api.types.is_scalar(x):
             return x
         elif isinstance(x, pd.Series):
             return Series(x)
@@ -478,7 +478,7 @@ def _call(self, x1, x2):
             df1, df2 = (x1, x2) if isinstance(x1, DATAFRAME_TYPE) else (x2, x1)
             setattr(self, '_object_type', ObjectType.dataframe)
             kw = self._calc_properties(df1, df2, axis=self.axis)
-            if not np.isscalar(df2):
+            if not pd.api.types.is_scalar(df2):
                 return self.new_dataframe([x1, x2], **kw)
             else:
                 return self.new_dataframe([df1], **kw)
@@ -486,7 +486,7 @@ def _call(self, x1, x2):
             s1, s2 = (x1, x2) if isinstance(x1, SERIES_TYPE) else (x2, x1)
             setattr(self, '_object_type', ObjectType.series)
             kw = self._calc_properties(s1, s2)
-            if not np.isscalar(s2):
+            if not pd.api.types.is_scalar(s2):
                 return self.new_series([x1, x2], **kw)
             else:
                 return self.new_series([s1], **kw)
diff --git a/mars/dataframe/base/reset_index.py b/mars/dataframe/base/reset_index.py
--- a/mars/dataframe/base/reset_index.py
+++ b/mars/dataframe/base/reset_index.py
@@ -96,11 +96,15 @@ def _tile_dataframe(cls, op):
         out_df = op.outputs[0]
         added_columns_num = len(out_df.dtypes) - len(in_df.dtypes)
         out_chunks = []
-        is_range_index = out_df.index_value.has_value()
+        index_has_value = out_df.index_value.has_value()
+        chunk_has_nan = any(np.isnan(s) for s in in_df.nsplits[0])
         cum_range = np.cumsum((0, ) + in_df.nsplits[0])
         for c in in_df.chunks:
-            if is_range_index:
-                index_value = parse_index(pd.RangeIndex(cum_range[c.index[0]], cum_range[c.index[0] + 1]))
+            if index_has_value:
+                if chunk_has_nan:
+                    index_value = parse_index(pd.RangeIndex(-1))
+                else:
+                    index_value = parse_index(pd.RangeIndex(cum_range[c.index[0]], cum_range[c.index[0] + 1]))
             else:
                 index_value = out_df.index_value
             if c.index[1] == 0:
@@ -116,8 +120,9 @@ def _tile_dataframe(cls, op):
                 new_chunk = chunk_op.new_chunk([c], shape=c.shape, index_value=index_value,
                                                index=c.index, columns_value=c.columns_value, dtypes=c.dtypes)
             out_chunks.append(new_chunk)
-        if not is_range_index and isinstance(out_df.index_value._index_value, IndexValue.RangeIndex):
-            out_chunks = standardize_range_index(out_chunks)
+        if not index_has_value or chunk_has_nan:
+            if isinstance(out_df.index_value._index_value, IndexValue.RangeIndex):
+                out_chunks = standardize_range_index(out_chunks)
         new_op = op.copy()
         columns_splits = list(in_df.nsplits[1])
         columns_splits[0] += added_columns_num
@@ -187,7 +192,7 @@ def _call_dataframe(self, a):
             empty_df.index = a.index_value.to_pandas()[:0]
             empty_df = empty_df.reset_index(level=self.level, col_level=self.col_level, col_fill=self.col_fill)
             shape = (a.shape[0], len(empty_df.columns))
-            columns_value = parse_index(empty_df.columns)
+            columns_value = parse_index(empty_df.columns, store_data=True)
             dtypes = empty_df.dtypes
             index_value = self._get_out_index(empty_df, shape)
         return self.new_dataframe([a], shape=shape, columns_value=columns_value,


Test patch:
diff --git a/mars/dataframe/arithmetic/tests/test_comparison.py b/mars/dataframe/arithmetic/tests/test_comparison.py
--- a/mars/dataframe/arithmetic/tests/test_comparison.py
+++ b/mars/dataframe/arithmetic/tests/test_comparison.py
@@ -14,6 +14,7 @@
 
 import operator
 import unittest
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
@@ -48,3 +49,11 @@ def testComp(self):
                                          columns=['a', 'b', 'c']))
             with self.assertRaises(ValueError):
                 op(df1, df4)
+
+        # test datetime
+        df = DataFrame(pd.DataFrame(pd.date_range('20130101', periods=6)))
+        for op in [operator.eq, operator.ne, operator.lt, operator.gt,
+                   operator.le, operator.ge]:
+            r_df = op(df, datetime(2013, 1, 2))
+            pd.testing.assert_index_equal(r_df.index_value.to_pandas(),
+                                          df.index_value.to_pandas())
diff --git a/mars/dataframe/base/tests/test_base.py b/mars/dataframe/base/tests/test_base.py
--- a/mars/dataframe/base/tests/test_base.py
+++ b/mars/dataframe/base/tests/test_base.py
@@ -192,6 +192,7 @@ def testResetIndex(self):
 
         self.assertEqual(df.shape, (4, 3))
         pd.testing.assert_series_equal(df.dtypes, r.dtypes)
+        pd.testing.assert_index_equal(df.columns_value.to_pandas(), r.columns)
 
         df2 = df.tiles()
 
diff --git a/mars/dataframe/base/tests/test_base_execution.py b/mars/dataframe/base/tests/test_base_execution.py
--- a/mars/dataframe/base/tests/test_base_execution.py
+++ b/mars/dataframe/base/tests/test_base_execution.py
@@ -204,6 +204,15 @@ def testResetIndexExecution(self):
         expected = (data1 + data2).reset_index()
         np.testing.assert_array_equal(result.to_numpy(), expected.to_numpy())
 
+        # case from https://github.com/mars-project/mars/issues/1286
+        data = pd.DataFrame(np.random.rand(10, 3), columns=list('abc'))
+        df = from_pandas_df(data, chunk_size=3)
+
+        r = df.sort_values('a').reset_index(drop=True)
+        result = self.executor.execute_dataframe(r, concat=True)[0]
+        expected = data.sort_values('a').reset_index(drop=True)
+        pd.testing.assert_frame_equal(result, expected)
+
     def testSeriesMapExecution(self):
         raw = pd.Series(np.arange(10))
         s = from_pandas_series(raw, chunk_size=7)