Repository: mars-project/mars

Issue:
[BUG] `df.describe()` failed when df has unknown shape and chunk size > 1
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

`df.describe()` failed when df has unknown shape and chunk size > 1

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```ipython
In [7]: df = md.DataFrame(mt.random.rand(10, 3), chunk_size=3)                  

In [8]: df[df[0] < .5].describe().execute()                                     
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-8-d9330a9157df> in <module>
----> 1 df[df[0] < .5].describe().execute()

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
    559 
    560     def execute(self, session=None, **kw):
--> 561         self._data.execute(session, **kw)
    562         return self
    563 

~/Workspace/mars/mars/core.py in execute(self, session, **kw)
    368 
    369         # no more fetch, thus just fire run
--> 370         session.run(self, **kw)
    371         # return Tileable or ExecutableTuple itself
    372         return self

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
    384         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
    385                           for t in tileables)
--> 386         result = self._sess.run(*tileables, **kw)
    387 
    388         for t in tileables:

~/Workspace/mars/mars/session.py in run(self, *tileables, **kw)
     97             # set number of running cores
     98             self.context.set_ncores(kw['n_parallel'])
---> 99             res = self._executor.execute_tileables(tileables, **kw)
    100             return res
    101 

~/Workspace/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Workspace/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Workspace/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose)
    817                 # build chunk graph, tile will be done during building
    818                 chunk_graph = chunk_graph_builder.build(
--> 819                     tileables, tileable_graph=tileable_graph)
    820                 tileable_graph = chunk_graph_builder.prev_tileable_graph
    821                 temp_result_keys = set(result_keys)

~/Workspace/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Workspace/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Workspace/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    340 
    341         chunk_graph = super().build(
--> 342             tileables, tileable_graph=tileable_graph)
    343         self._iterative_chunk_graphs.append(chunk_graph)
    344         if len(self._interrupted_ops) == 0:

~/Workspace/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Workspace/mars/mars/utils.py in inner(*args, **kwargs)
    471     def inner(*args, **kwargs):
    472         with build_mode():
--> 473             return func(*args, **kwargs)
    474     return inner
    475 

~/Workspace/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    253                     # for further execution
    254                     partial_tiled_chunks = \
--> 255                         self._on_tile_failure(tileable_data.op, exc_info)
    256                     if partial_tiled_chunks is not None and \
    257                             len(partial_tiled_chunks) > 0:

~/Workspace/mars/mars/tiles.py in inner(op, exc_info)
    292                     on_tile_failure(op, exc_info)
    293                 else:
--> 294                     raise exc_info[1].with_traceback(exc_info[2]) from None
    295         return inner
    296 

~/Workspace/mars/mars/tiles.py in build(self, tileables, tileable_graph)
    233                 continue
    234             try:
--> 235                 tiled = self._tile(tileable_data, tileable_graph)
    236                 tiled_op.add(tileable_data.op)
    237                 for t, td in zip(tileable_data.op.outputs, tiled):

~/Workspace/mars/mars/tiles.py in _tile(self, tileable_data, tileable_graph)
    328         if any(inp.op in self._interrupted_ops for inp in tileable_data.inputs):
    329             raise TilesError('Tile fail due to failure of inputs')
--> 330         return super()._tile(tileable_data, tileable_graph)
    331 
    332     @kernel_mode

~/Workspace/mars/mars/tiles.py in _tile(self, tileable_data, tileable_graph)
    191                 t._nsplits = o.nsplits
    192         elif on_tile is None:
--> 193             tds[0]._inplace_tile()
    194         else:
    195             tds = on_tile(tileable_data.op.outputs, tds)

~/Workspace/mars/mars/core.py in _inplace_tile(self)
    160 
    161     def _inplace_tile(self):
--> 162         return handler.inplace_tile(self)
    163 
    164     def __getattr__(self, attr):

~/Workspace/mars/mars/tiles.py in inplace_tile(self, to_tile)
    126         if not to_tile.is_coarse():
    127             return to_tile
--> 128         dispatched = self.dispatch(to_tile.op)
    129         self._assign_to([d.data for d in dispatched], to_tile.op.outputs)
    130         return to_tile

~/Workspace/mars/mars/utils.py in _wrapped(*args, **kwargs)
    383                 _kernel_mode.eager = False
    384             _kernel_mode.eager_count = enter_eager_count + 1
--> 385             return func(*args, **kwargs)
    386         finally:
    387             _kernel_mode.eager_count -= 1

~/Workspace/mars/mars/tiles.py in dispatch(self, op)
    113             return self._handlers[op_cls](op)
    114         try:
--> 115             return op_cls.tile(op)
    116         except NotImplementedError as ex:
    117             cause = ex

~/Workspace/mars/mars/dataframe/base/describe.py in tile(cls, op)
     91             return cls._tile_series(op)
     92         else:
---> 93             return cls._tile_dataframe(op)
     94 
     95     @classmethod

~/Workspace/mars/mars/dataframe/base/describe.py in _tile_dataframe(cls, op)
    136         names = index.tolist()
    137 
--> 138         df = df.rechunk({1: df.shape[1]})
    139         df = df[columns]
    140 

~/Workspace/mars/mars/dataframe/base/rechunk.py in rechunk(a, chunk_size, threshold, chunk_size_limit)
     88     else:
     89         itemsize = a.dtype.itemsize
---> 90     chunk_size = get_nsplits(a, chunk_size, itemsize)
     91     if chunk_size == a.nsplits:
     92         return a

~/Workspace/mars/mars/tensor/rechunk/core.py in get_nsplits(tileable, new_chunk_size, itemsize)
     36         chunk_size = new_chunk_size
     37 
---> 38     return decide_chunk_sizes(tileable.shape, chunk_size, itemsize)
     39 
     40 

~/Workspace/mars/mars/tensor/utils.py in decide_chunk_sizes(shape, chunk_size, itemsize)
    546         raise ValueError("chunks have more dimensions than input tensor")
    547     if nleft == 0:
--> 548         return normalize_chunk_sizes(shape, tuple(chunk_size[j] for j in range(len(shape))))
    549 
    550     max_chunk_size = options.chunk_store_limit

~/Workspace/mars/mars/tensor/utils.py in normalize_chunk_sizes(shape, chunk_size)
     64             if sum(chunk) != size:
     65                 raise ValueError('chunks shape should be of the same length, '
---> 66                                  'got shape: %s, chunks: %s' % (size, chunk))
     67             chunk_sizes.append(chunk)
     68         else:

ValueError: chunks shape should be of the same length, got shape: nan, chunks: (nan, nan, nan, nan)
```




Discussion:

Resolving patch:
diff --git a/mars/dataframe/base/describe.py b/mars/dataframe/base/describe.py
--- a/mars/dataframe/base/describe.py
+++ b/mars/dataframe/base/describe.py
@@ -135,7 +135,6 @@ def _tile_dataframe(cls, op):
         # ['count', 'mean', 'std', 'min', {percentiles}, 'max']
         names = index.tolist()
 
-        df = df.rechunk({1: df.shape[1]})
         df = df[columns]
 
         values = [None] * 6
diff --git a/mars/dataframe/utils.py b/mars/dataframe/utils.py
--- a/mars/dataframe/utils.py
+++ b/mars/dataframe/utils.py
@@ -21,7 +21,7 @@
 import pandas as pd
 from pandas.core.dtypes.cast import find_common_type
 
-from ..core import Entity
+from ..core import Entity, ExecutableTuple
 from ..lib.mmh3 import hash as mmh_hash
 from ..tensor.utils import dictify_chunk_size, normalize_chunk_sizes
 from ..utils import tokenize, sbytes
@@ -749,8 +749,10 @@ def fetch_corner_data(df_or_series, session=None) -> pd.DataFrame:
     if index_size is None:
         return df_or_series.fetch(session=session)
     else:
-        head_data = iloc(df_or_series)[:index_size].fetch(session=session)
-        tail_data = iloc(df_or_series)[-index_size:].fetch(session=session)
+        head = iloc(df_or_series)[:index_size]
+        tail = iloc(df_or_series)[-index_size:]
+        head_data, tail_data = \
+            ExecutableTuple([head, tail]).fetch(session=session)
         return pd.concat([head_data, tail_data], axis='index')
 
 


Test patch:
diff --git a/mars/dataframe/base/tests/test_base_execution.py b/mars/dataframe/base/tests/test_base_execution.py
--- a/mars/dataframe/base/tests/test_base_execution.py
+++ b/mars/dataframe/base/tests/test_base_execution.py
@@ -26,7 +26,7 @@
 from mars.dataframe.datasource.index import from_pandas as from_pandas_index
 from mars.session import new_session
 from mars.tensor import tensor
-from mars.tests.core import TestBase, require_cudf, ExecutorForTest
+from mars.tests.core import TestBase, require_cudf
 from mars.utils import lazy_import
 
 cudf = lazy_import('cudf', globals=globals())
@@ -35,7 +35,7 @@
 class Test(TestBase):
     def setUp(self):
         super().setUp()
-        self.executor = ExecutorForTest()
+        self.ctx, self.executor = self._create_test_context()
 
     @require_cudf
     def testToGPUExecution(self):
@@ -337,6 +337,16 @@ def testDescribeExecution(self):
         with self.assertRaises(ValueError):
             df.describe(percentiles=[1.1])
 
+        # test input dataframe which has unknown shape
+        with self.ctx:
+            df = from_pandas_df(df_raw, chunk_size=3)
+            df2 = df[df['a'] < 0.5]
+            r = df2.describe()
+
+            result = self.executor.execute_tileables([r])[0]
+            expected = df_raw[df_raw['a'] < 0.5].describe()
+            pd.testing.assert_frame_equal(result, expected)
+
     def testDataFrameFillNAExecution(self):
         df_raw = pd.DataFrame(np.nan, index=range(0, 20), columns=list('ABCDEFGHIJ'))
         for _ in range(20):