Repository: mars-project/mars

Issue:
[BUG] Inconsistent behavior with Pandas when do arithmetic calculation
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
The behavior is inconsistent with Pandas when add a DataFrame with tensor.

**To Reproduce**
```Python
In [72]: (md.DataFrame(np.ones((10, 3)), index=range(1,11)) + np.random.rand(10, 3)).execute()                                                    
Out[72]: 
           0         1         2
0        NaN       NaN       NaN
1   1.638449  1.014959  1.932828
2   1.165521  1.270455  1.246919
3   1.155571  1.772896  1.093757
4   1.137783  1.669508  1.155838
5   1.011088  1.060680  1.827767
6   1.340579  1.618841  1.063207
7   1.111091  1.241154  1.206726
8   1.965057  1.007527  1.948954
9   1.711392  1.753188  1.475779
10       NaN       NaN       NaN
```

**Expected behavior**
``` Python
In [75]: pd.DataFrame(np.ones((10, 3)), index=range(1,11)) + np.random.rand(10, 3)                                                                
Out[75]: 
           0         1         2
1   1.550521  1.489118  1.567964
2   1.659091  1.662107  1.488283
3   1.998321  1.201712  1.882578
4   1.012231  1.172180  1.039455
5   1.663685  1.080299  1.165124
6   1.815481  1.856006  1.053097
7   1.218592  1.481979  1.782585
8   1.746447  1.745898  1.523206
9   1.859547  1.972220  1.368852
10  1.177347  1.158729  1.311812

```



Discussion:

Resolving patch:
diff --git a/mars/dataframe/arithmetic/core.py b/mars/dataframe/arithmetic/core.py
--- a/mars/dataframe/arithmetic/core.py
+++ b/mars/dataframe/arithmetic/core.py
@@ -16,14 +16,17 @@
 import copy
 
 import numpy as np
-
+import pandas as pd
 from pandas.core.dtypes.cast import find_common_type
 
+from ...tensor.datasource import tensor as astensor
 from ...serialize import AnyField, Float64Field
+
+from ...tensor.core import TENSOR_TYPE, ChunkData, Chunk
 from ...utils import classproperty
 from ..align import align_series_series, align_dataframe_series, align_dataframe_dataframe
 from ..core import DATAFRAME_TYPE, SERIES_TYPE, DATAFRAME_CHUNK_TYPE, SERIES_CHUNK_TYPE
-from ..operands import DataFrameOperandMixin, DataFrameOperand, ObjectType
+from ..operands import DataFrameOperandMixin, ObjectType, DataFrameOperand
 from ..initializer import Series, DataFrame
 from ..utils import parse_index, infer_dtypes, infer_index_value
 
@@ -189,7 +192,7 @@ def _tile_scalar(cls, op):
                                              columns_value=getattr(chunk, 'columns_value'))
             else:
                 out_chunk = out_op.new_chunk([chunk], shape=chunk.shape, index=chunk.index, dtype=chunk.dtype,
-                                            index_value=chunk.index_value, name=getattr(chunk, 'name'))
+                                             index_value=chunk.index_value, name=getattr(chunk, 'name'))
             out_chunks.append(out_chunk)
 
         new_op = op.copy()
@@ -200,6 +203,44 @@ def _tile_scalar(cls, op):
             return new_op.new_dataframes(op.inputs, df.shape, nsplits=tileable.nsplits, dtypes=df.dtypes,
                                          index_value=df.index_value, columns_value=df.columns_value, chunks=out_chunks)
 
+    @classmethod
+    def _tile_with_tensor(cls, op):
+        rhs_is_tensor = isinstance(op.rhs, TENSOR_TYPE)
+        tensor, other = (op.rhs, op.lhs) if rhs_is_tensor else (op.lhs, op.rhs)
+        if tensor.shape == other.shape:
+            tensor = tensor.rechunk(other.nsplits).single_tiles()
+        else:
+            # shape differs only when dataframe add 1-d tensor, we need rechunk on columns axis.
+            rechunk_size = other.nsplits[1] if op.axis == 'columns' or op.axis == 1 else other.nsplits[0]
+            tensor = tensor.rechunk((rechunk_size,)).single_tiles()
+
+        out_chunks = []
+        for out_index in itertools.product(*(map(range, other.chunk_shape))):
+            tensor_chunk = tensor.cix[out_index[:tensor.ndim]]
+            other_chunk = other.cix[out_index]
+            out_op = op.copy().reset_key()
+            inputs = [other_chunk, tensor_chunk] if rhs_is_tensor else [tensor_chunk, other_chunk]
+            if isinstance(other_chunk, DATAFRAME_CHUNK_TYPE):
+                out_chunk = out_op.new_chunk(inputs, shape=other_chunk.shape, index=other_chunk.index,
+                                             dtypes=other_chunk.dtypes,
+                                             index_value=other_chunk.index_value,
+                                             columns_value=other.columns_value)
+            else:
+                out_chunk = out_op.new_chunk(inputs, shape=other_chunk.shape, index=other_chunk.index,
+                                             dtype=other_chunk.dtype,
+                                             index_value=other_chunk.index_value,
+                                             name=other_chunk.name)
+            out_chunks.append(out_chunk)
+
+        new_op = op.copy()
+        if isinstance(other, SERIES_TYPE):
+            return new_op.new_seriess(op.inputs, other.shape, nsplits=other.nsplits, dtype=other.dtype,
+                                      index_value=other.index_value, name=other.name, chunks=out_chunks)
+        else:
+            return new_op.new_dataframes(op.inputs, other.shape, nsplits=other.nsplits, dtypes=other.dtypes,
+                                         index_value=other.index_value, columns_value=other.columns_value,
+                                         chunks=out_chunks)
+
     @classmethod
     def tile(cls, op):
         if len(op.inputs) < 2:
@@ -212,6 +253,8 @@ def tile(cls, op):
             return cls._tile_dataframe_series(op)
         elif isinstance(op.inputs[0], SERIES_TYPE) and isinstance(op.inputs[1], DATAFRAME_TYPE):
             return cls._tile_series_dataframe(op)
+        elif isinstance(op.inputs[0], TENSOR_TYPE) or isinstance(op.inputs[1], TENSOR_TYPE):
+            return cls._tile_with_tensor(op)
 
     @classmethod
     def execute(cls, ctx, op):
@@ -222,7 +265,7 @@ def execute(cls, ctx, op):
                 func_name = getattr(cls, '_rfunc_name')
             else:
                 func_name = getattr(cls, '_func_name')
-        elif np.isscalar(op.lhs):
+        elif np.isscalar(op.lhs) or isinstance(op.lhs, np.ndarray):
             df = ctx[op.rhs.key]
             other = op.lhs
             func_name = getattr(cls, '_rfunc_name')
@@ -242,12 +285,14 @@ def _operator(self):
 
     @classmethod
     def _calc_properties(cls, x1, x2=None, axis='columns'):
-        if isinstance(x1, (DATAFRAME_TYPE, DATAFRAME_CHUNK_TYPE)) and (x2 is None or np.isscalar(x2)):
+        if isinstance(x1, (DATAFRAME_TYPE, DATAFRAME_CHUNK_TYPE)) \
+                and (x2 is None or np.isscalar(x2) or isinstance(x2, TENSOR_TYPE)):
             # FIXME infer the dtypes of result df properly
             return {'shape': x1.shape, 'dtypes': x1.dtypes,
                     'columns_value': x1.columns_value, 'index_value': x1.index_value}
 
-        if isinstance(x1, (SERIES_TYPE, SERIES_CHUNK_TYPE)) and (x2 is None or np.isscalar(x2)):
+        if isinstance(x1, (SERIES_TYPE, SERIES_CHUNK_TYPE)) \
+                and (x2 is None or np.isscalar(x2) or isinstance(x2, TENSOR_TYPE)):
             dtype = find_common_type([x1.dtype, type(x2)])
             return {'shape': x1.shape, 'dtype': dtype, 'index_value': x1.index_value}
 
@@ -336,13 +381,15 @@ def _calc_properties(cls, x1, x2=None, axis='columns'):
         raise NotImplementedError('Unknown combination of parameters')
 
     def _new_chunks(self, inputs, kws=None, **kw):
-        if len(inputs) == 1:
-            properties = self._calc_properties(*inputs)
+        property_inputs = [inp for inp in inputs if isinstance(inp, (DATAFRAME_CHUNK_TYPE, SERIES_CHUNK_TYPE))]
+        if len(property_inputs) == 1:
+            properties = self._calc_properties(*property_inputs)
         else:
-            df1 = inputs[0] if isinstance(inputs[0], DATAFRAME_CHUNK_TYPE) else inputs[1]
-            df2 = inputs[1] if isinstance(inputs[0], DATAFRAME_CHUNK_TYPE) else inputs[0]
+            df1, df2 = property_inputs if isinstance(property_inputs[0], DATAFRAME_CHUNK_TYPE) else \
+                reversed(property_inputs)
             properties = self._calc_properties(df1, df2, axis=self.axis)
 
+        inputs = [inp for inp in inputs if isinstance(inp, (Chunk, ChunkData))]
         shapes = [properties.pop('shape')]
         shapes.extend(kw_item.pop('shape') for kw_item in kws or ())
         if 'shape' in kw:
@@ -360,33 +407,56 @@ def _new_chunks(self, inputs, kws=None, **kw):
     def _process_input(x):
         if isinstance(x, (DATAFRAME_TYPE, SERIES_TYPE)) or np.isscalar(x):
             return x
-        elif isinstance(x, (list, tuple)) or getattr(x, 'ndim', None) == 1:
+        elif isinstance(x, pd.Series):
             return Series(x)
-        elif getattr(x, 'ndim', None) == 2:
+        elif isinstance(x, pd.DataFrame):
             return DataFrame(x)
-        else:
-            raise NotImplementedError("Don't support type {}".format(type(x)))
+        elif isinstance(x, (list, tuple, np.ndarray, TENSOR_TYPE)):
+            return astensor(x)
+        raise NotImplementedError
+
+    def _check_inputs(self, x1, x2):
+        if isinstance(x1, TENSOR_TYPE) or isinstance(x2, TENSOR_TYPE):
+            tensor, other = (x1, x2) if isinstance(x1, TENSOR_TYPE) else (x2, x1)
+            if isinstance(other, DATAFRAME_TYPE):
+                if self.axis == 'index' or self.axis == 0:
+                    other_shape = tuple(reversed(other.shape))
+                else:
+                    other_shape = other.shape
+                if tensor.ndim == 2 and tensor.shape != other_shape:
+                    raise ValueError(
+                        'Unable to coerce to DataFrame, shape must be {}: given {}'.format(other_shape, tensor.shape))
+                elif tensor.ndim == 1 and tensor.shape[0] != other_shape[1]:
+                    raise ValueError(
+                        'Unable to coerce to Series, length must be {}: given {}'.format(other_shape[1], tensor.shape[0]))
+                elif tensor.ndim > 2:
+                    raise ValueError('Unable to coerce to Series/DataFrame, dim must be <= 2')
+            if isinstance(other, SERIES_TYPE):
+                if tensor.ndim == 1 and (tensor.shape[0] != other.shape[0]):
+                    raise ValueError(
+                        'Unable to coerce to Series, length must be {}: given {}'.format(other.shape[0], tensor.shape[0]))
+                elif tensor.ndim > 1:
+                    raise ValueError('Unable to coerce to Series, dim must be 1')
 
     def _call(self, x1, x2):
+        self._check_inputs(x1, x2)
         if isinstance(x1, DATAFRAME_TYPE) or isinstance(x2, DATAFRAME_TYPE):
-            df1 = x1 if isinstance(x1, DATAFRAME_TYPE) else x2
-            df2 = x2 if isinstance(x1, DATAFRAME_TYPE) else x1
+            df1, df2 = (x1, x2) if isinstance(x1, DATAFRAME_TYPE) else (x2, x1)
             setattr(self, '_object_type', ObjectType.dataframe)
             kw = self._calc_properties(df1, df2, axis=self.axis)
-            if isinstance(df2, (DATAFRAME_TYPE, SERIES_TYPE)):
+            if not np.isscalar(df2):
                 return self.new_dataframe([x1, x2], **kw)
             else:
                 return self.new_dataframe([df1], **kw)
         if isinstance(x1, SERIES_TYPE) or isinstance(x2, SERIES_TYPE):
-            s1 = x1 if isinstance(x1, SERIES_TYPE) else x2
-            s2 = x2 if isinstance(x1, SERIES_TYPE) else x1
+            s1, s2 = (x1, x2) if isinstance(x1, SERIES_TYPE) else (x2, x1)
             setattr(self, '_object_type', ObjectType.series)
             kw = self._calc_properties(s1, s2)
-            if isinstance(s2, SERIES_TYPE):
+            if not np.isscalar(s2):
                 return self.new_series([x1, x2], **kw)
             else:
                 return self.new_series([s1], **kw)
-        raise NotImplementedError('Only support add dataframe or scalar for now')
+        raise NotImplementedError('Only support add dataframe, series or scalar for now')
 
     def __call__(self, x1, x2):
         x1 = self._process_input(x1)


Test patch:
diff --git a/mars/dataframe/arithmetic/tests/test_arithmetic.py b/mars/dataframe/arithmetic/tests/test_arithmetic.py
--- a/mars/dataframe/arithmetic/tests/test_arithmetic.py
+++ b/mars/dataframe/arithmetic/tests/test_arithmetic.py
@@ -998,6 +998,28 @@ def testSeriesAndScalar(self):
             self.assertIsInstance(cr.inputs[0].op, SeriesDataSource)
             self.assertEqual(cr.op.lhs, 789)
 
+    def testCheckInputs(self):
+        data = pd.DataFrame(np.random.rand(10, 3))
+        df = from_pandas(data)
+
+        with self.assertRaises(ValueError):
+            _ = df + np.random.rand(5, 3)
+
+        with self.assertRaises(ValueError):
+            _ = df + np.random.rand(10)
+
+        with self.assertRaises(ValueError):
+            _ = df + np.random.rand(10, 3, 2)
+
+        data = pd.Series(np.random.rand(10))
+        series = from_pandas_series(data)
+
+        with self.assertRaises(ValueError):
+            _ = series + np.random.rand(5, 3)
+
+        with self.assertRaises(ValueError):
+            _ = series + np.random.rand(5)
+
 
 class TestUnary(TestBase):
     def testAbs(self):
diff --git a/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py b/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py
--- a/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py
+++ b/mars/dataframe/arithmetic/tests/test_arithmetic_execution.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 import operator
 
 import numpy as np
@@ -488,16 +487,6 @@ def testWithPlainValue(self):
         expected = getattr(data1, self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10), axis=0)
         pd.testing.assert_frame_equal(expected, result)
 
-        r = getattr(df1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0)
-        result = self.executor.execute_dataframe(r, concat=True)[0]
-        expected = getattr(data1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0)
-        pd.testing.assert_frame_equal(expected, result)
-
-        r = getattr(df1, self.func_name)(from_array(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])), axis=0)
-        result = self.executor.execute_dataframe(r, concat=True)[0]
-        expected = getattr(data1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0)
-        pd.testing.assert_frame_equal(expected, result)
-
         r = getattr(s1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
         result = self.executor.execute_dataframe(r, concat=True)[0]
         expected = getattr(data1[2], self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
@@ -508,6 +497,22 @@ def testWithPlainValue(self):
         expected = getattr(data1[2], self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
         pd.testing.assert_series_equal(expected, result)
 
+        # specify index, not the default range index
+        data1 = pd.DataFrame(np.random.rand(10, 7), index=np.arange(5, 15),
+                             columns=[4, 1, 3, 2, 5, 6, 7])
+        df1 = from_pandas(data1, chunk_size=6)
+        s1 = df1[2]
+
+        r = getattr(df1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0)
+        result = self.executor.execute_dataframe(r, concat=True)[0]
+        expected = getattr(data1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0)
+        pd.testing.assert_frame_equal(expected, result)
+
+        r = getattr(df1, self.func_name)(from_array(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])), axis=0)
+        result = self.executor.execute_dataframe(r, concat=True)[0]
+        expected = getattr(data1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0)
+        pd.testing.assert_frame_equal(expected, result)
+
         r = getattr(s1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
         result = self.executor.execute_dataframe(r, concat=True)[0]
         expected = getattr(data1[2], self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
@@ -518,25 +523,6 @@ def testWithPlainValue(self):
         expected = getattr(data1[2], self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
         pd.testing.assert_series_equal(expected, result)
 
-    @unittest.expectedFailure
-    def testWithPlainValueUnaligned(self):
-        # When adding dataframe with a sequence value, pandas treats the sequence
-        # as a series using the index_value of the dataframe.
-        #
-        # In mars we cannot do such things because the index_value is not stored.
-        # We also cannot split the sequence using the nsplits of the dataframe since
-        # in many cases the shape of the dataframe chunks is np.nan.
-        #
-        # We record this case as `expectedFailure`.
-        data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10),
-                             columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
-        df1 = from_pandas(data1, chunk_size=6)
-
-        r = getattr(df1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], axis=1)
-        result = self.executor.execute_dataframe(r, concat=True)[0]
-        expected = getattr(data1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], axis=0)
-        pd.testing.assert_frame_equal(expected, result)
-
 
 class TestUnary(TestBase):
     def setUp(self):