Repository: mars-project/mars

Issue:
[BUG] some operation must be executed explicitly before
**Describe the bug**
Some operation must be executed explicitly before, the code is shown as follows.
By adding `df.execute()`, the error disappears.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version: 3.8.3
2. The version of Mars you use: pymars[distributed]==0.6.0
3. Versions of crucial packages, such as numpy, scipy and protobuf: numpy==1.19.4, scipy==1.5.4, protobuf==3.14.0, pyarrow==2.0.0
4. Minimized code and full stack of the error.
```python
import mars.dataframe as md
df = md.read_parquet("data.parquet")
# df.execute()
df.to_parquet("result-*.parquet").execute()
```
```bash
Traceback (most recent call last):
  File "demo.py", line 7, in <module>
    df.to_parquet("result-*.parquet").execute()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 643, in execute
    return run()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 639, in run
    self.data.execute(session, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 379, in execute
    return run()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 374, in run
    session.run(self, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/session.py", line 499, in run
    result = self._sess.run(*tileables, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/session.py", line 108, in run
    res = self._executor.execute_tileables(tileables, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/executor.py", line 860, in execute_tileables
    chunk_graph = chunk_graph_builder.build(
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 347, in build
    chunk_graph = super().build(
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 262, in build
    self._on_tile_failure(tileable_data.op, exc_info)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 301, in inner
    raise exc_info[1].with_traceback(exc_info[2]) from None
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 242, in build
    tiled = self._tile(tileable_data, tileable_graph)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 337, in _tile
    return super()._tile(tileable_data, tileable_graph)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 201, in _tile
    tds[0]._inplace_tile()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 168, in _inplace_tile
    return handler.inplace_tile(self)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 136, in inplace_tile
    dispatched = self.dispatch(to_tile.op)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 119, in dispatch
    tiled = op_cls.tile(op)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/datastore/to_parquet.py", line 104, in tile
    in_df = in_df.rechunk({1: in_df.shape[1]})._inplace_tile()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/base/rechunk.py", line 96, in rechunk
    chunk_size = get_nsplits(a, chunk_size, itemsize)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tensor/rechunk/core.py", line 38, in get_nsplits
    return decide_chunk_sizes(tileable.shape, chunk_size, itemsize)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tensor/utils.py", line 551, in decide_chunk_sizes
    return normalize_chunk_sizes(shape, tuple(chunk_size[j] for j in range(len(shape))))
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tensor/utils.py", line 66, in normalize_chunk_sizes
    raise ValueError('chunks shape should be of the same length, '
ValueError: chunks shape should be of the same length, got shape: nan, chunks: (nan,)
```

```python
from mars.learn.contrib import xgboost as xgb
import mars.dataframe as md

df = md.read_csv("data.csv")
# df.execute()
model = xgb.XGBClassifier()
model.load_model('classifer.model')
print(model.predict_proba(df).execute())
```
```bash
Traceback (most recent call last):
  File "test_model.py", line 36, in <module>
    model.predict_proba(df).execute()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/learn/contrib/xgboost/classifier.py", line 78, in predict_proba
    return predict(self.get_booster(), data, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/learn/contrib/xgboost/predict.py", line 159, in predict
    result = op()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/learn/contrib/xgboost/predict.py", line 65, in __call__
    shape = (len(self._data),)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/core.py", line 1145, in __len__
    return len(self._data)
TypeError: 'float' object cannot be interpreted as an integer
```
**Expected behavior**
Just one `execute` operation at the end without explicitly adding `df.execute()`.


Discussion:
Thank you for your report, we will fix it ASAP.

Resolving patch:
diff --git a/mars/dataframe/datastore/to_parquet.py b/mars/dataframe/datastore/to_parquet.py
--- a/mars/dataframe/datastore/to_parquet.py
+++ b/mars/dataframe/datastore/to_parquet.py
@@ -20,6 +20,8 @@
 from ...filesystem import open_file
 from ...serialize import KeyField, AnyField, StringField, ListField, \
     BoolField, DictField
+from ...tiles import TilesError
+from ...utils import check_chunks_unknown_shape
 from ..operands import DataFrameOperand, DataFrameOperandMixin
 from ..utils import parse_index
 from ..datasource.read_parquet import check_engine
@@ -101,6 +103,7 @@ def tile(cls, op):
         out_df = op.outputs[0]
 
         # make sure only 1 chunk on the column axis
+        check_chunks_unknown_shape([in_df], TilesError)
         in_df = in_df.rechunk({1: in_df.shape[1]})._inplace_tile()
 
         out_chunks = []
diff --git a/mars/learn/contrib/xgboost/predict.py b/mars/learn/contrib/xgboost/predict.py
--- a/mars/learn/contrib/xgboost/predict.py
+++ b/mars/learn/contrib/xgboost/predict.py
@@ -60,9 +60,9 @@ def __call__(self):
         if num_class is not None:
             num_class = int(num_class)
         if num_class is not None:
-            shape = (len(self._data), num_class)
+            shape = (self._data.shape[0], num_class)
         else:
-            shape = (len(self._data),)
+            shape = (self._data.shape[0],)
         if self.output_types[0] == OutputType.tensor:
             # tensor
             return self.new_tileable([self._data], shape=shape, dtype=np.dtype(np.float32),
@@ -89,10 +89,10 @@ def tile(cls, op):
             chunk_op = op.copy().reset_key()
             chunk_index = (in_chunk.index[0],)
             if op.model.attr('num_class'):
-                chunk_shape = (len(in_chunk), int(op.model.attr('num_class')))
+                chunk_shape = (in_chunk.shape[0], int(op.model.attr('num_class')))
                 chunk_index += (0,)
             else:
-                chunk_shape = (len(in_chunk),)
+                chunk_shape = (in_chunk.shape[0],)
             if op.output_types[0] == OutputType.tensor:
                 out_chunk = chunk_op.new_chunk([in_chunk], shape=chunk_shape,
                                                dtype=out.dtype,


Test patch:
diff --git a/mars/dataframe/datastore/tests/test_datastore_execute.py b/mars/dataframe/datastore/tests/test_datastore_execute.py
--- a/mars/dataframe/datastore/tests/test_datastore_execute.py
+++ b/mars/dataframe/datastore/tests/test_datastore_execute.py
@@ -197,6 +197,11 @@ def testToParquetArrowExecution(self):
             result = result.sort_index()
             pd.testing.assert_frame_equal(result, raw)
 
+            # test read_parquet then to_parquet
+            read_df = md.read_parquet(path)
+            r = read_df.to_parquet(path)
+            self.executor.execute_dataframes([r])
+
             # test partition_cols
             path = os.path.join(base_path, 'out-partitioned')
             r = df.to_parquet(path, partition_cols=['col3'])
diff --git a/mars/learn/contrib/xgboost/tests/test_classifier.py b/mars/learn/contrib/xgboost/tests/test_classifier.py
--- a/mars/learn/contrib/xgboost/tests/test_classifier.py
+++ b/mars/learn/contrib/xgboost/tests/test_classifier.py
@@ -105,6 +105,12 @@ def testLocalClassifier(self):
         self.assertEqual(prediction.ndim, 1)
         self.assertEqual(prediction.shape[0], len(self.X))
 
+        # test predict data with unknown shape
+        X2 = X[X[:, 0] > 0.1].astype(mt.int32)
+        prediction = classifier.predict(X2)
+
+        self.assertEqual(prediction.ndim, 1)
+
         classifier = XGBClassifier(verbosity=1, n_estimators=2)
         with self.assertRaises(TypeError):
             classifier.fit(X, y, wrong_param=1)