Repository: chainer/chainer

Issue:
Result of numpy.ndarray + Variable is not intuitive
In current implementation, results of following operations are (possibly broadcasted) element addition : 
- `Variable + scalar`
- `scalar + Variable`
- `Variable + numpy.ndarray` (as long as Variable.data + numpy.ndarray is valid)

It is natural to suppose `numpy.ndarray + Variable` works in the same way.
But return value of `numpy.ndarray + Variable` is numpy.ndarray whose elements are Variable as shown in the following example.

```
In [75]: aa = Variable(np.random.uniform(0,1,3))
In [76]: b = np.random.uniform(0,1,3)
In [77]: b + aa
Out[77]: 
array([<chainer.variable.Variable object at 0x7f2a5e1384d0>,
       <chainer.variable.Variable object at 0x7f2a5e138710>,
       <chainer.variable.Variable object at 0x7f2a5e138950>], dtype=object)
In [78]: (b + aa)[0].data
Out[78]: array([ 0.90954313,  0.60971617,  1.05675581])
```

I think it would be better `(b+aa).data` equals to `b + aa.data`



Discussion:

Resolving patch:
diff --git a/chainer/cuda.py b/chainer/cuda.py
--- a/chainer/cuda.py
+++ b/chainer/cuda.py
@@ -663,6 +663,27 @@ def copy_async(array, out=None, out_device=None, stream=None):
 
     return out
 
+# ------------------------------------------------------------------------------
+# Add comparison of `__array_priority__` to GPUArray binary operator
+# ------------------------------------------------------------------------------
+def _wrap_operation(obj, op):
+    op_name = '__{}__'.format(op)
+    rop_name = '__r{}__'.format(op)
+    raw_op = getattr(obj, op_name)
+    def new_op(self, other):
+        rop =  getattr(other, rop_name, None)
+        if rop is None:
+            return raw_op(self, other)
+        self_pri  = getattr(self,  '__array_priority__', 0.0)
+        other_pri = getattr(other, '__array_priority__', 0.0)
+        if self_pri >= other_pri:
+            return raw_op(self, other)
+        return rop(self)
+    setattr(obj, op_name, new_op)
+
+if available:
+    for op in ('add', 'sub', 'mul', 'div', 'pow'):
+        _wrap_operation(GPUArray, op)
 
 # ------------------------------------------------------------------------------
 # Interprocess communication
diff --git a/chainer/functions/basic_math.py b/chainer/functions/basic_math.py
--- a/chainer/functions/basic_math.py
+++ b/chainer/functions/basic_math.py
@@ -1,5 +1,6 @@
 import math
 import numpy
+from numbers import Number
 from chainer import cuda, Function, Variable
 
 # ------------------------------------------------------------------------------
@@ -135,10 +136,16 @@ def backward_cpu(self, x, gy):
 
     def backward_gpu(self, x, gy):
         gx = cuda.empty_like(x[0])
-        cuda.elementwise(
-            'float* gx, const float* x, const float* gy, float value',
-            'gx[i] = -value * gy[i] / (x[i] * x[i])',
-            'div_from_const_bwd')(gx, x[0], gy[0], self.value)
+        if isinstance(self.value, Number):
+            cuda.elementwise(
+                'float* gx, const float* x, const float* gy, const float value',
+                'gx[i] = -value * gy[i] / (x[i] * x[i])',
+                'div_from_const_bwd')(gx, x[0], gy[0], self.value)
+        else:
+            cuda.elementwise(
+                'float* gx, const float* x, const float* gy, const float* value',
+                'gx[i] = -value[i] * gy[i] / (x[i] * x[i])',
+                'div_from_const_bwd')(gx, x[0], gy[0], self.value)
         return gx,
 
 def rdiv(lhs, rhs):  # rhs / lhs
@@ -180,10 +187,16 @@ def backward_cpu(self, x, gy):
 
     def backward_gpu(self, x, gy):
         gx = cuda.empty_like(x[0])
-        cuda.elementwise(
-            'float* gx, const float* x, const float* gy, float value',
-            'gx[i] = value * __powf(x[i], value - 1) * gy[i]',
-            'pow_var_const_bwd')(gx, x[0], gy[0], self.value)
+        if isinstance(self.value, Number):
+            cuda.elementwise(
+                'float* gx, const float* x, const float* gy, const float value',
+                'gx[i] = value * __powf(x[i], value - 1) * gy[i]',
+                'pow_var_const_bwd')(gx, x[0], gy[0], self.value)
+        else:
+            cuda.elementwise(
+                'float* gx, const float* x, const float* gy, const float* value',
+                'gx[i] = value[i] * __powf(x[i], value[i] - 1) * gy[i]',
+                'pow_var_const_bwd')(gx, x[0], gy[0], self.value)
         return gx,
 
 def pow(lhs, rhs):  # lhs ** rhs
@@ -201,21 +214,32 @@ def forward_cpu(self, x):
 
     def forward_gpu(self, x):
         y = cuda.empty_like(x[0])
-        cuda.elementwise('float* y, const float* x, float value',
-                         'y[i] = __powf(value, x[i])',
-                         'pow_const_var_fwd')(y, x[0], self.value)
+        if isinstance(self.value, Number):
+            cuda.elementwise('float* y, const float* x, const float value',
+                             'y[i] = __powf(value, x[i])',
+                             'pow_const_var_fwd')(y, x[0], self.value)
+        else:
+            cuda.elementwise('float* y, const float* x, const float *value',
+                             'y[i] = __powf(value[i], x[i])',
+                             'pow_const_var_fwd')(y, x[0], self.value)
         return y,
 
     def backward_cpu(self, x, gy):
-        return math.log(self.value) * self.y * gy[0],
+        return numpy.log(self.value) * self.y * gy[0],
 
     def backward_gpu(self, x, gy):
-        logv = math.log(self.value)
         gx = cuda.empty_like(x[0])
-        cuda.elementwise(
-            'float* gx, const float* x, const float* gy, float value, float logv',
-            'gx[i] = logv * __powf(value, x[i]) * gy[i]',
-            'pow_const_var_bwd')(gx, x[0], gy[0], self.value, logv)
+        if isinstance(self.value, Number):
+            logv = math.log(self.value)
+            cuda.elementwise(
+                'float* gx, const float* x, const float* gy, const float value, const float logv',
+                'gx[i] = logv * __powf(value, x[i]) * gy[i]',
+                'pow_const_var_bwd')(gx, x[0], gy[0], self.value, logv)
+        else:
+            cuda.elementwise(
+                'float* gx, const float* x, const float* gy, const float* value',
+                'gx[i] = __logf(value[i]) * __powf(value[i], x[i]) * gy[i]',
+                'pow_const_var_bwd')(gx, x[0], gy[0], self.value)
         return gx,
 
 def rpow(lhs, rhs):  # rhs ** lhs
diff --git a/chainer/variable.py b/chainer/variable.py
--- a/chainer/variable.py
+++ b/chainer/variable.py
@@ -181,3 +181,5 @@ def add_cand(cand):
             for var in func.inputs:
                 add_cand(var.creator)
             func.unchain()
+
+    __array_priority__ = 200


Test patch:
diff --git a/tests/functions_tests/test_basic_math.py b/tests/functions_tests/test_basic_math.py
--- a/tests/functions_tests/test_basic_math.py
+++ b/tests/functions_tests/test_basic_math.py
@@ -196,53 +196,59 @@ def setUp(self):
         self.gy = numpy.random.uniform(-1, 1, (3, 2)).astype(numpy.float32)
         self.value = numpy.random.uniform(-1, 1, (3, 2)).astype(numpy.float32)
 
-    def check_forward(self, op, x_data, gpu):
+    def check_forward(self, op, x_data, gpu, positive):
         value = self.value
+        if positive:
+            value = numpy.abs(value)
+        v = value
         if gpu:
-            value = cuda.to_gpu(value)
+            v = cuda.to_gpu(v)
         x = Variable(x_data)
-        y = op(x, value)
-        assert_allclose(op(self.x, self.value), y.data, atol=1e-6, rtol=1e-6)
+        y = op(x, v)
+        assert_allclose(op(self.x, value), y.data, atol=1e-6, rtol=1e-6)
 
-    def forward_cpu(self, op):
-        self.check_forward(op, self.x, False)
+    def forward_cpu(self, op, positive=False):
+        self.check_forward(op, self.x, False, positive)
 
-    # TODO(delta2323): Commented tests check if op(array, Variable)
-    # work where op is radd/rsub and so on.
-    # Currently these fails. See issue #43 for details.
-    # (https://github.com/pfnet/chainer/issues/43)
     def test_add_forward_cpu(self):  self.forward_cpu(lambda x, y: x + y)
-    # def test_radd_forward_cpu(self): self.forward_cpu(lambda x, y: y + x)
+    def test_radd_forward_cpu(self): self.forward_cpu(lambda x, y: y + x)
     def test_sub_forward_cpu(self):  self.forward_cpu(lambda x, y: x - y)
-    # def test_rsub_forward_cpu(self): self.forward_cpu(lambda x, y: y - x)
+    def test_rsub_forward_cpu(self): self.forward_cpu(lambda x, y: y - x)
     def test_mul_forward_cpu(self):  self.forward_cpu(lambda x, y: x * y)
-    # def test_rmul_forward_cpu(self): self.forward_cpu(lambda x, y: y * x)
+    def test_rmul_forward_cpu(self): self.forward_cpu(lambda x, y: y * x)
     def test_div_forward_cpu(self):  self.forward_cpu(lambda x, y: x / y)
-    # def test_rdiv_forward_cpu(self): self.forward_cpu(lambda x, y: y / x)
+    def test_rdiv_forward_cpu(self): self.forward_cpu(lambda x, y: y / x)
     def test_pow_forward_cpu(self):  self.forward_cpu(lambda x, y: x ** y)
-    # def test_rpow_forward_cpu(self): self.forward_cpu(lambda x, y: y ** x)
+    def test_rpow_forward_cpu(self): self.forward_cpu(lambda x, y: y ** x, positive=True)
 
-    def forward_gpu(self, op):
-        self.check_forward(op, to_gpu(self.x), True)
+    def forward_gpu(self, op, positive=False):
+        self.check_forward(op, to_gpu(self.x), True, positive)
 
     @attr.gpu
     def test_add_forward_gpu(self):  self.forward_gpu(lambda x, y: x + y)
-    # def test_radd_forward_gpu(self): self.forward_gpu(lambda x, y: y + x)
+    @attr.gpu
+    def test_radd_forward_gpu(self): self.forward_gpu(lambda x, y: y + x)
     @attr.gpu
     def test_sub_forward_gpu(self):  self.forward_gpu(lambda x, y: x - y)
-    # def test_rsub_forward_gpu(self): self.forward_gpu(lambda x, y: y - x)
+    @attr.gpu
+    def test_rsub_forward_gpu(self): self.forward_gpu(lambda x, y: y - x)
     @attr.gpu
     def test_mul_forward_gpu(self):  self.forward_gpu(lambda x, y: x * y)
-    # def test_rmul_forward_gpu(self): self.forward_gpu(lambda x, y: y * x)
+    @attr.gpu
+    def test_rmul_forward_gpu(self): self.forward_gpu(lambda x, y: y * x)
     @attr.gpu
     def test_div_forward_gpu(self):  self.forward_gpu(lambda x, y: x / y)
-    # def test_rdiv_forward_gpu(self): self.forward_gpu(lambda x, y: y / x)
+    @attr.gpu
+    def test_rdiv_forward_gpu(self): self.forward_gpu(lambda x, y: y / x)
     @attr.gpu
     def test_pow_forward_gpu(self):  self.forward_gpu(lambda x, y: x ** y)
-    # def test_rpow_forward_gpu(self): self.forward_gpu(lambda x, y: y ** x)
+    @attr.gpu
+    def test_rpow_forward_gpu(self): self.forward_gpu(lambda x, y: y ** x, positive=True)
 
-    def check_backward(self, op, x_data, y_grad, gpu):
+    def check_backward(self, op, x_data, y_grad, gpu, positive):
         value = self.value
+        if positive:
+            value = numpy.abs(value)
         if gpu:
             value = cuda.to_gpu(value)
         x = Variable(x_data)
@@ -256,39 +262,41 @@ def check_backward(self, op, x_data, y_grad, gpu):
 
         assert_allclose(gx, x.grad, atol=1e-4, rtol=1e-4)
 
-    def backward_cpu(self, op):
-        self.check_backward(op, self.x, self.gy, False)
+    def backward_cpu(self, op, positive=False):
+        self.check_backward(op, self.x, self.gy, False, positive)
 
     def test_add_backward_cpu(self):  self.backward_cpu(lambda x, y: x + y)
-    # def test_radd_backward_cpu(self): self.backward_cpu(lambda x, y: y + x)
+    def test_radd_backward_cpu(self): self.backward_cpu(lambda x, y: y + x)
     def test_sub_backward_cpu(self):  self.backward_cpu(lambda x, y: x - y)
-    # def test_rsub_backward_cpu(self): self.backward_cpu(lambda x, y: y - x)
+    def test_rsub_backward_cpu(self): self.backward_cpu(lambda x, y: y - x)
     def test_mul_backward_cpu(self):  self.backward_cpu(lambda x, y: x * y)
-    # def test_rmul_backward_cpu(self): self.backward_cpu(lambda x, y: y * x)
+    def test_rmul_backward_cpu(self): self.backward_cpu(lambda x, y: y * x)
     def test_div_backward_cpu(self):  self.backward_cpu(lambda x, y: x / y)
-    # def test_rdiv_backward_cpu(self): self.backward_cpu(lambda x, y: y / x)
+    def test_rdiv_backward_cpu(self): self.backward_cpu(lambda x, y: y / x)
     def test_pow_backward_cpu(self):  self.backward_cpu(lambda x, y: x ** y)
-    # def test_rpow_backward_cpu(self): self.backward_cpu(lambda x, y: y ** x)
+    def test_rpow_backward_cpu(self): self.backward_cpu(lambda x, y: y ** x, positive=True)
 
-    def backward_gpu(self, op):
-        self.check_backward(op, to_gpu(self.x), to_gpu(self.gy), True)
+    def backward_gpu(self, op, positive=False):
+        self.check_backward(op, to_gpu(self.x), to_gpu(self.gy), True, positive)
 
     @attr.gpu
     def test_add_backward_gpu(self):  self.backward_gpu(lambda x, y: x + y)
-    # def test_radd_backward_gpu(self): self.backward_gpu(lambda x, y: y + x)
+    @attr.gpu
+    def test_radd_backward_gpu(self): self.backward_gpu(lambda x, y: y + x)
     @attr.gpu
     def test_sub_backward_gpu(self):  self.backward_gpu(lambda x, y: x - y)
-    # TODO(delta2323): backward_gpu of SubFromConstant(resp. DivFromConstant,
-    # PowVarConst(pow) and PowConstVar(rpow)) assumes constant value is scalar.
-    # So corresponding test for rsub (resp. rsub, rdiv, pow, and rpow) fails.
     @attr.gpu
     def test_mul_backward_gpu(self):  self.backward_gpu(lambda x, y: x * y)
-    # def test_rmul_backward_gpu(self): self.backward_gpu(lambda x, y: y * x)
+    @attr.gpu
+    def test_rmul_backward_gpu(self): self.backward_gpu(lambda x, y: y * x)
     @attr.gpu
     def test_div_backward_gpu(self):  self.backward_gpu(lambda x, y: x / y)
-    # def test_rdiv_backward_gpu(self): self.backward_gpu(lambda x, y: y / x)
-    # def test_pow_backward_gpu(self):  self.backward_gpu(lambda x, y: x ** y)
-    # def test_rpow_backward_gpu(self): self.backward_gpu(lambda x, y: y ** x)
+    @attr.gpu
+    def test_rdiv_backward_gpu(self): self.backward_gpu(lambda x, y: y / x)
+    @attr.gpu
+    def test_pow_backward_gpu(self):  self.backward_gpu(lambda x, y: x ** y)
+    @attr.gpu
+    def test_rpow_backward_gpu(self): self.backward_gpu(lambda x, y: y ** x, positive=True)
 
 class TestUnaryFunctions(TestCase):
     def setUp(self):