Repository: chainer/chainer

Issue:
Curse of float64
Adding `int` to 0-dim array causes a `float64` value.

```
>>> (numpy.array(1, numpy.float32) + 1).dtype
dtype('float64')
```

When its dimension is greater than 0, this problem isn't happen.

```
>>> (numpy.array([1], numpy.float32) + 1).dtype
dtype('float32’)
```

Only functions in `basic_math` accept 0-dim array as an input value.
These functions only forward input data to numpy, and it may causes `float64` problem.

We need to check types of arguments and when they contains `float32` and standard python types like `int` or `float`, we need to convert results to `float32` as results of calculation are `float64`.

For example, this code is dangerous:

```
loss = softmax_cross_entropy(...) / 10
loss.backward()
```



Discussion:
```
>>> x = chainer.Variable(numpy.array(1, numpy.float32))
>>> (x + 1).data.dtype
dtype('float64')
```


Resolving patch:
diff --git a/chainer/functions/basic_math.py b/chainer/functions/basic_math.py
--- a/chainer/functions/basic_math.py
+++ b/chainer/functions/basic_math.py
@@ -26,6 +26,13 @@ def _convert_value_to_string(value):
             'value must be float, ndarray, GPUArray, or Variable')
 
 
+def _force_type(dtype, value):
+    if numpy.isscalar(value):
+        return dtype.type(value)
+    else:
+        return value
+
+
 class Neg(function.Function):
 
     @property
@@ -88,7 +95,7 @@ def label(self):
         return '_ + %s' % _convert_value_to_string(self.value)
 
     def forward(self, x):
-        return utils.force_array(x[0] + self.value),
+        return utils.force_array(x[0] + _force_type(x[0].dtype, self.value)),
 
     def backward(self, x, gy):
         return gy[0],
@@ -129,7 +136,7 @@ def label(self):
         return '%s - _' % _convert_value_to_string(self.value)
 
     def forward(self, x):
-        return utils.force_array(self.value - x[0]),
+        return utils.force_array(_force_type(x[0].dtype, self.value) - x[0]),
 
     def backward(self, x, gy):
         return utils.force_array(-gy[0]),
@@ -177,10 +184,10 @@ def label(self):
         return '_ * %s' % _convert_value_to_string(self.value)
 
     def forward(self, x):
-        return utils.force_array(self.value * x[0]),
+        return utils.force_array(_force_type(x[0].dtype, self.value) * x[0]),
 
     def backward(self, x, gy):
-        return utils.force_array(self.value * gy[0]),
+        return utils.force_array(_force_type(gy[0].dtype, self.value) * gy[0]),
 
 
 def mul(lhs, rhs):  # lhs * rhs
@@ -232,10 +239,11 @@ def label(self):
         return '_ / %s' % _convert_value_to_string(self.value)
 
     def forward(self, x):
-        return utils.force_array(self.value / x[0]),
+        return utils.force_array(_force_type(x[0].dtype, self.value) / x[0]),
 
     def backward_cpu(self, x, gy):
-        return utils.force_array(-self.value * gy[0] / (x[0] ** 2)),
+        value = _force_type(gy[0].dtype, self.value)
+        return utils.force_array(-value * gy[0] / (x[0] ** 2)),
 
     def backward_gpu(self, x, gy):
         gx = cuda.empty_like(x[0])
@@ -306,10 +314,11 @@ def label(self):
         return '_ ** %s' % _convert_value_to_string(self.value)
 
     def forward(self, x):
-        return utils.force_array(x[0] ** self.value),
+        return utils.force_array(x[0] ** _force_type(x[0].dtype, self.value)),
 
     def backward_cpu(self, x, gy):
-        gx = self.value * (x[0] ** (self.value - 1)) * gy[0]
+        val_1 = _force_type(x[0].dtype, self.value - 1)
+        gx = _force_type(x[0].dtype, self.value) * (x[0] ** val_1) * gy[0]
         return utils.force_array(gx),
 
     def backward_gpu(self, x, gy):
@@ -349,7 +358,7 @@ def label(self):
         return '%s ** _' % _convert_value_to_string(self.value)
 
     def forward_cpu(self, x):
-        self.y = utils.force_array(self.value ** x[0])
+        self.y = utils.force_array(_force_type(x[0].dtype, self.value) ** x[0])
         return self.y,
 
     def forward_gpu(self, x):
@@ -365,7 +374,8 @@ def forward_gpu(self, x):
         return y,
 
     def backward_cpu(self, x, gy):
-        return utils.force_array(numpy.log(self.value) * self.y * gy[0]),
+        value = _force_type(gy[0].dtype, self.value)
+        return utils.force_array(numpy.log(value) * self.y * gy[0]),
 
     def backward_gpu(self, x, gy):
         gx = cuda.empty_like(x[0])


Test patch:
diff --git a/tests/functions_tests/test_basic_math.py b/tests/functions_tests/test_basic_math.py
--- a/tests/functions_tests/test_basic_math.py
+++ b/tests/functions_tests/test_basic_math.py
@@ -220,6 +220,65 @@ def make_data(self):
         return x1, x2, gy
 
 
+class TestBinaryOpConstant(unittest.TestCase):
+
+    def _test_constant(self, func):
+        x_data = numpy.array(1, numpy.float32)
+        x1 = chainer.Variable(x_data)
+        y1 = func(x1, 1)
+        self.assertEqual(y1.data.dtype, numpy.float32)
+        y1.backward()
+        self.assertEqual(x1.grad.dtype, numpy.float32)
+
+        x2 = chainer.Variable(x_data)
+        y2 = func(x2, 1.0)
+        self.assertEqual(y2.data.dtype, numpy.float32)
+        y2.backward()
+        self.assertEqual(x2.grad.dtype, numpy.float32)
+
+        x3 = chainer.Variable(x_data)
+        y3 = func(x3, numpy.int64(1))
+        self.assertEqual(y3.data.dtype, numpy.float32)
+        y3.backward()
+        self.assertEqual(x3.grad.dtype, numpy.float32)
+
+        x4 = chainer.Variable(x_data)
+        y4 = func(x4, numpy.float64(1.0))
+        self.assertEqual(y4.data.dtype, numpy.float32)
+        y4.backward()
+        self.assertEqual(x4.grad.dtype, numpy.float32)
+
+    def test_add_constnt(self):
+        self._test_constant(lambda x, y: x + y)
+
+    def test_radd_constnt(self):
+        self._test_constant(lambda x, y: y + x)
+
+    def test_sub_constnt(self):
+        self._test_constant(lambda x, y: x - y)
+
+    def test_rsub_constnt(self):
+        self._test_constant(lambda x, y: y - x)
+
+    def test_mul_constnt(self):
+        self._test_constant(lambda x, y: x * y)
+
+    def test_rmul_constnt(self):
+        self._test_constant(lambda x, y: y * x)
+
+    def test_div_constnt(self):
+        self._test_constant(lambda x, y: x / y)
+
+    def test_rdiv_constnt(self):
+        self._test_constant(lambda x, y: y / x)
+
+    def test_pow_constnt(self):
+        self._test_constant(lambda x, y: x ** y)
+
+    def test_rpow_constnt(self):
+        self._test_constant(lambda x, y: y ** x)
+
+
 class VariableConstantOpTestBase(object):
 
     def make_date(self):