Repository: mars-project/mars

Issue:
[BUG] Multiplication encountered ValueError
**Describe the bug**
When multiplying with power of certain DataFrame, ValueError raises with message "Length of passed values is x, index implies y".

**To Reproduce**
```python
>>> import numpy as np
>>> import pandas as pd
>>> import mars.dataframe as md
>>> rs = np.random.RandomState(0)
>>> raw_df = rs.rand(20, 10)
>>> raw_df = pd.DataFrame(np.where(raw_df > 0.4, raw_df, np.nan), columns=list('ABCDEFGHIJ'))
>>> df = md.DataFrame(raw_df, chunk_size=6)
>>> raw_df2 = rs.rand(20, 10)
>>> raw_df2 = pd.DataFrame(np.where(raw_df2 > 0.4, raw_df2, np.nan), columns=list('ABCDEFGHIJ'))
>>> df2 = md.DataFrame(raw_df2, chunk_size=4)
>>> sumv = (df ** 2).mul(df2, axis=1, fill_value=0).sum(axis=0).execute()
Traceback (most recent call last):
  File "/Users/wenjun.swj/miniconda3/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3417, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-16-f7addf55e6d4>", line 1, in <module>
    sumv = (df ** 2).mul(df2, axis=1, fill_value=0).sum(axis=0).execute()
  File "/Users/wenjun.swj/Code/mars/mars/dataframe/arithmetic/__init__.py", line 118, in call_df_fill
    return func(df, other, axis=axis, level=level, fill_value=fill_value)
  File "/Users/wenjun.swj/Code/mars/mars/dataframe/arithmetic/multiply.py", line 48, in mul
    return op(df, other)
  File "/Users/wenjun.swj/Code/mars/mars/dataframe/arithmetic/core.py", line 499, in __call__
    return self._call(x1, x2)
  File "/Users/wenjun.swj/Code/mars/mars/dataframe/arithmetic/core.py", line 479, in _call
    kw = self._calc_properties(df1, df2, axis=self.axis)
  File "/Users/wenjun.swj/Code/mars/mars/dataframe/arithmetic/core.py", line 326, in _calc_properties
    dtypes = pd.Series([infer_dtype(dt1, dt2, cls._operator) for dt1, dt2
  File "/Users/wenjun.swj/miniconda3/lib/python3.8/site-packages/pandas/core/series.py", line 313, in __init__
    raise ValueError(
ValueError: Length of passed values is 10, index implies 11.
```


Discussion:

Resolving patch:
diff --git a/mars/dataframe/arithmetic/core.py b/mars/dataframe/arithmetic/core.py
--- a/mars/dataframe/arithmetic/core.py
+++ b/mars/dataframe/arithmetic/core.py
@@ -27,7 +27,7 @@
 from ..operands import DataFrameOperandMixin, DataFrameOperand
 from ..initializer import Series, DataFrame
 from ..ufunc.tensor import TensorUfuncMixin
-from ..utils import parse_index, infer_dtypes, infer_dtype, infer_index_value
+from ..utils import parse_index, infer_dtypes, infer_dtype, infer_index_value, build_empty_df
 
 
 class DataFrameBinOp(DataFrameOperand):
@@ -301,7 +301,7 @@ def _calc_properties(cls, x1, x2=None, axis='columns'):
             if x2 is None:
                 dtypes = x1.dtypes
             elif pd.api.types.is_scalar(x2):
-                dtypes = infer_dtypes(x1.dtypes, pd.Series(np.array(x2).dtype), cls._operator)
+                dtypes = cls._operator(build_empty_df(x1.dtypes), x2).dtypes
             elif x1.dtypes is not None and isinstance(x2, TENSOR_TYPE):
                 dtypes = pd.Series(
                     [infer_dtype(dt, x2.dtype, cls._operator) for dt in x1.dtypes],
diff --git a/mars/dataframe/datasource/read_parquet.py b/mars/dataframe/datasource/read_parquet.py
--- a/mars/dataframe/datasource/read_parquet.py
+++ b/mars/dataframe/datasource/read_parquet.py
@@ -268,6 +268,7 @@ def read_parquet(path, engine: str = "auto", columns=None,
                  **kwargs):
     """
     Load a parquet object from the file path, returning a DataFrame.
+
     Parameters
     ----------
     path : str, path object or file-like object
@@ -298,10 +299,12 @@ def read_parquet(path, engine: str = "auto", columns=None,
         Options for storage connection.
     **kwargs
         Any additional kwargs are passed to the engine.
+
     Returns
     -------
     Mars DataFrame
     """
+
     if not isinstance(path, list):
         file_path = glob(path, storage_options=storage_options)[0]
     else:
diff --git a/mars/dataframe/utils.py b/mars/dataframe/utils.py
--- a/mars/dataframe/utils.py
+++ b/mars/dataframe/utils.py
@@ -476,14 +476,15 @@ def build_df(df_obj, fill_value=1, size=1):
     empty_df = build_empty_df(df_obj.dtypes, index=df_obj.index_value.to_pandas()[:0])
     dtypes = empty_df.dtypes
     record = [_generate_value(dtype, fill_value) for dtype in dtypes]
-    if isinstance(empty_df.index, pd.MultiIndex):
-        index = tuple(_generate_value(level.dtype, fill_value) for level in empty_df.index.levels)
-        empty_df = empty_df.reindex(
-            index=pd.MultiIndex.from_tuples([index], names=empty_df.index.names))
-        empty_df.iloc[0] = record
-    else:
-        index = _generate_value(empty_df.index.dtype, fill_value)
-        empty_df.loc[index] = record
+    if len(record) != 0:  # columns is empty in some cases
+        if isinstance(empty_df.index, pd.MultiIndex):
+            index = tuple(_generate_value(level.dtype, fill_value) for level in empty_df.index.levels)
+            empty_df = empty_df.reindex(
+                index=pd.MultiIndex.from_tuples([index], names=empty_df.index.names))
+            empty_df.iloc[0] = record
+        else:
+            index = _generate_value(empty_df.index.dtype, fill_value)
+            empty_df.loc[index] = record
 
     empty_df = pd.concat([empty_df] * size)
     # make sure dtypes correct for MultiIndex


Test patch:
diff --git a/mars/dataframe/arithmetic/tests/test_arithmetic.py b/mars/dataframe/arithmetic/tests/test_arithmetic.py
--- a/mars/dataframe/arithmetic/tests/test_arithmetic.py
+++ b/mars/dataframe/arithmetic/tests/test_arithmetic.py
@@ -1058,6 +1058,10 @@ def testDataFrameAndScalar(self):
         result3 = getattr(df, self.rfunc_name)(1)
         result4 = self.func(df, 1)
         result5 = self.func(1, df)
+
+        expected = self.func(data, 2)
+        pd.testing.assert_series_equal(result.dtypes, expected.dtypes)
+
         pd.testing.assert_index_equal(result.columns_value.to_pandas(), data.columns)
         self.assertIsInstance(result.index_value.value, IndexValue.Int64Index)
 
diff --git a/mars/dataframe/reduction/tests/test_reduction_execute.py b/mars/dataframe/reduction/tests/test_reduction_execute.py
--- a/mars/dataframe/reduction/tests/test_reduction_execute.py
+++ b/mars/dataframe/reduction/tests/test_reduction_execute.py
@@ -174,6 +174,14 @@ def testDataFrameReduction(self):
             self.compute(data, axis='index', numeric_only=True),
             self.executor.execute_dataframe(reduction_df, concat=True)[0])
 
+        data1 = pd.DataFrame(np.random.rand(10, 10), columns=[str(i) for i in range(10)])
+        data2 = pd.DataFrame(np.random.rand(10, 10), columns=[str(i) for i in range(10)])
+        df = from_pandas_df(data1, chunk_size=5) + from_pandas_df(data2, chunk_size=6)
+        reduction_df = self.compute(df)
+        pd.testing.assert_series_equal(
+            self.compute(data1 + data2).sort_index(),
+            self.executor.execute_dataframe(reduction_df, concat=True)[0].sort_index())
+
     @require_cudf
     def testGPUExecution(self):
         pdf = pd.DataFrame(np.random.rand(30, 3), columns=list('abc'))