Repository: mars-project/mars

Issue:
[BUG] Arithmetic failed for tensor on GPU
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Arithmetic failed for tensor on GPU.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [13]: x = mt.random.rand(10, 10, gpu=True)                                                                                                                                                                      

In [14]: (x + 1).execute()                                                                                                                                                                                         
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-14-60cf72259190> in <module>
----> 1 (x + 1).execute()

~/mars/mars/tensor/core.py in execute(self, session, **kw)
    450 
    451     def execute(self, session=None, **kw):
--> 452         return self._data.execute(session, **kw)
    453 
    454 

~/mars/mars/core.py in execute(self, session, **kw)
    582         if session is None:
    583             session = Session.default_or_local()
--> 584         return session.run(self, **kw)
    585 
    586     def fetch(self, session=None, **kw):

~/mars/mars/session.py in run(self, *tileables, **kw)
    150         tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
    151                           for t in tileables)
--> 152         result = self._sess.run(*tileables, **kw)
    153 
    154         for t in tileables:

~/mars/mars/session.py in run(self, *tileables, **kw)
     61         if 'n_parallel' not in kw:
     62             kw['n_parallel'] = cpu_count()
---> 63         res = self._executor.execute_tileables(tileables, **kw)
     64         return res
     65 

~/mars/mars/utils.py in _wrapped(*args, **kwargs)
    393                 _kernel_mode.eager = False
    394             _kernel_mode.eager_count = enter_eager_count + 1
--> 395             return func(*args, **kwargs)
    396         finally:
    397             _kernel_mode.eager_count -= 1

~/mars/mars/executor.py in execute_tileables(self, tileables, fetch, n_parallel, n_thread, print_progress, mock, compose)
    562         self.execute_graph(graph, result_keys, n_parallel=n_parallel or n_thread,
    563                            print_progress=print_progress, mock=mock,
--> 564                            chunk_result=chunk_result)
    565 
    566         self._chunk_result.update(chunk_result)

~/mars/mars/executor.py in execute_graph(self, graph, keys, n_parallel, print_progress, mock, no_intermediate, compose, retval, chunk_result)
    494                                          mock_max_memory=self._mock_max_memory,
    495                                          fetch_keys=fetch_keys, no_intermediate=no_intermediate)
--> 496         res = graph_execution.execute(retval)
    497         self._mock_max_memory = max(self._mock_max_memory, graph_execution._mock_max_memory)
    498         if mock:

~/mars/mars/executor.py in execute(self, retval)
    387         # wait until all the futures completed
    388         for future in executed_futures:
--> 389             future.result()
    390 
    391         if retval:

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
    423                 raise CancelledError()
    424             elif self._state == FINISHED:
--> 425                 return self.__get_result()
    426 
    427             self._condition.wait(timeout)

~/miniconda3/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
    382     def __get_result(self):
    383         if self._exception:
--> 384             raise self._exception
    385         else:
    386             return self._result

~/miniconda3/lib/python3.7/concurrent/futures/thread.py in run(self)
     55 
     56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
     58         except BaseException as exc:
     59             self.future.set_exception(exc)

~/mars/mars/executor.py in _execute_operand(self, op)
    265             # so we pass the first operand's first output to Executor.handle
    266             first_op = ops[0]
--> 267             Executor.handle(first_op, results, self._mock)
    268 
    269             # update maximal memory usage during execution

~/mars/mars/executor.py in handle(cls, op, results, mock)
    450             runner = getattr(op, method_name)
    451         try:
--> 452             return runner(results, op)
    453         except NotImplementedError:
    454             for op_cls in mapper.keys():

~/mars/mars/tensor/arithmetic/core.py in execute(cls, ctx, op)
    112 
    113             with np.errstate(**op.err):
--> 114                 ctx[op.outputs[0].key] = _handle_out_dtype(func(lhs, rhs, **kw), op.dtype)
    115 
    116 

cupy/core/_kernel.pyx in cupy.core._kernel.ufunc.__call__()

TypeError: Wrong arguments {'order': 'K'}
```

**Additional context**

As now `gpu` property would have value if it's deterministic mentioned in #577 , we may not specify the `parallel` parameter as the cpu count, if the tileable object is on gpu, we may specify `parallel` as 1 instead.


Discussion:

Resolving patch:
diff --git a/mars/tensor/arithmetic/core.py b/mars/tensor/arithmetic/core.py
--- a/mars/tensor/arithmetic/core.py
+++ b/mars/tensor/arithmetic/core.py
@@ -91,13 +91,25 @@ def check_inputs(self, inputs):
                 "Binary operand's inputs should less than or equal 4, got {0}".format(len(inputs)))
 
     @classmethod
-    def execute(cls, ctx, op):
+    def _execute_gpu(cls, op, xp, lhs, rhs, **kw):
+        func_name = getattr(cls, '_func_name')
+        r = getattr(xp, func_name)(lhs, rhs, **kw)
+        out_order = op.outputs[0].order.value
+        if xp.isfortran(r) != (out_order == 'F'):
+            r = xp.array(r, order=out_order)
+        return r
+
+    @classmethod
+    def _execute_cpu(cls, op, xp, lhs, rhs, **kw):
+        kw['order'] = op.order
         func_name = getattr(cls, '_func_name')
+        return getattr(xp, func_name)(lhs, rhs, **kw)
+
+    @classmethod
+    def execute(cls, ctx, op):
         inputs, device_id, xp = as_same_device(
             [ctx[c.key] for c in op.inputs], device=op.device, ret_extra=True)
 
-        func = getattr(xp, func_name)
-
         with device(device_id):
             kw = {'casting': op.casting} if op.out else {}
 
@@ -108,10 +120,13 @@ def execute(cls, ctx, op):
                 kw['out'] = next(inputs_iter).copy()
             if op.where:
                 kw['where'] = next(inputs_iter)
-            kw['order'] = op.order
 
             with np.errstate(**op.err):
-                ctx[op.outputs[0].key] = _handle_out_dtype(func(lhs, rhs, **kw), op.dtype)
+                if op.is_gpu():
+                    ret = cls._execute_gpu(op, xp, lhs, rhs, **kw)
+                else:
+                    ret = cls._execute_cpu(op, xp, lhs, rhs, **kw)
+                ctx[op.outputs[0].key] = _handle_out_dtype(ret, op.dtype)
 
 
 class TensorBinOp(TensorOperand, TensorBinOpMixin):
@@ -265,12 +280,25 @@ def _get_func(cls, xp):
         func_name = getattr(cls, '_func_name')
         return getattr(xp, func_name)
 
+    @classmethod
+    def _execute_gpu(cls, op, xp, inp, **kw):
+        r = cls._get_func(xp)(inp, **kw)
+        out_order = op.outputs[0].order.value
+        if xp.isfortran(r) != (out_order == 'F'):
+            r = xp.array(r, order=out_order)
+        return r
+
+    @classmethod
+    def _execute_cpu(cls, op, xp, inp, **kw):
+        if op.order != 'K':
+            kw['order'] = op.order
+        return cls._get_func(xp)(inp, **kw)
+
     @classmethod
     def execute(cls, ctx, op):
         inputs, device_id, xp = as_same_device(
             [ctx[c.key] for c in op.inputs], device=op.device, ret_extra=True)
 
-        func = cls._get_func(xp)
         with device(device_id):
             kw = {'casting': op.casting} if op.out else {}
 
@@ -280,11 +308,13 @@ def execute(cls, ctx, op):
                 inputs, kw['out'] = inputs[:-1], inputs[-1].copy()
             elif op.where:
                 inputs, kw['where'] = inputs[:-1], inputs[-1]
-            if op.order != 'K':
-                kw['order'] = op.order
 
             with np.errstate(**op.err):
-                ctx[op.outputs[0].key] = _handle_out_dtype(func(inputs[0], **kw), op.dtype)
+                if op.is_gpu():
+                    ret = cls._execute_gpu(op, xp, inputs[0], **kw)
+                else:
+                    ret = cls._execute_cpu(op, xp, inputs[0], **kw)
+                ctx[op.outputs[0].key] = _handle_out_dtype(ret, op.dtype)
 
 
 class TensorUnaryOp(TensorOperand, TensorUnaryOpMixin):


Test patch:
diff --git a/mars/tensor/arithmetic/tests/test_arithmetic_execution.py b/mars/tensor/arithmetic/tests/test_arithmetic_execution.py
--- a/mars/tensor/arithmetic/tests/test_arithmetic_execution.py
+++ b/mars/tensor/arithmetic/tests/test_arithmetic_execution.py
@@ -24,6 +24,7 @@
 from mars.tensor.datasource import ones, tensor, zeros
 from mars.tensor.arithmetic import add, cos, truediv, frexp, \
     modf, clip, isclose
+from mars.tensor.array_utils import cp
 from mars.config import option_context
 
 
@@ -650,3 +651,16 @@ def testSetGetImagExecution(self):
         expected.imag = np.array([9, 8, 7])
 
         np.testing.assert_equal(res, expected)
+
+    @unittest.skipIf(cp is None, 'cupy not installed')
+    def testCupy(self):
+        a_data = np.random.rand(10, 10)
+        b_data = np.random.rand(10, 10)
+
+        a = tensor(a_data, gpu=True, chunk_size=3)
+        b = tensor(b_data, gpu=True, chunk_size=3)
+        res_binary = self.executor.execute_tensor((a + b), concat=True)[0]
+        np.testing.assert_array_equal(res_binary.get(), (a_data + b_data))
+
+        res_unary = self.executor.execute_tensor(cos(a), concat=True)[0]
+        np.testing.assert_array_almost_equal(res_unary.get(), np.cos(a_data))