Repository: chainer/chainer

Issue:
1.7.0 debug mode bug
Hi,

I tested the debug model on one of my existing models and got:

```
Traceback (most recent call last):
  File "./train.py", line 53, in <module>
    loss.backward()
  File "/home/eddie/envs/deep/local/lib/python2.7/site-packages/chainer/variable.py", line 355, in backward
    for gx in gxs):
  File "/home/eddie/envs/deep/local/lib/python2.7/site-packages/chainer/variable.py", line 355, in <genexpr>
    for gx in gxs):
TypeError: ufunc 'isnan' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''
```

Here gx is 

```
[[ 0.0001472   0.00014604  0.00015193 ...,  0.00015092  0.0001509
   0.00014626]
 [ 0.00014647  0.0001442   0.00015269 ...,  0.00015215  0.00015142
   0.00014504]
 [ 0.00014855  0.00014735  0.00015144 ...,  0.00015121  0.00015061
   0.00014711]
 ...,
 [ 0.00014409  0.00014379  0.00015338 ...,  0.00015306  0.00015099
   0.00014355]
 [ 0.00014459  0.00014202  0.00015292 ...,  0.00015324  0.00015215
   0.00014102]
 [ 0.00014621  0.00014452  0.00015207 ...,  0.00015231  0.00015252
   0.00014397]]
```

composed of np.float32



Discussion:
Sorry, this is a critical bug. I'll fix quickly.


Resolving patch:
diff --git a/chainer/function.py b/chainer/function.py
--- a/chainer/function.py
+++ b/chainer/function.py
@@ -121,7 +121,8 @@ def __call__(self, *inputs):
             hook.forward_postprocess(self, in_data)
 
         if chainer.is_debug():
-            if any(cuda.get_array_module(out).isnan(out).any()
+            if any(out.dtype.kind == 'f' and
+                   cuda.get_array_module(out).isnan(out).any()
                    for out in outputs):
                 msg = 'NaN is detected on forward computation'
                 raise RuntimeError(msg)
diff --git a/chainer/variable.py b/chainer/variable.py
--- a/chainer/variable.py
+++ b/chainer/variable.py
@@ -349,7 +349,8 @@ def add_cand(cand):
                 hook.backward_postprocess(func, in_data, out_grad)
 
             if chainer.is_debug():
-                if any(cuda.get_array_module(gx).isnan(gx).any()
+                if any(gx is not None and
+                       cuda.get_array_module(gx).isnan(gx).any()
                        for gx in gxs):
                     msg = 'NaN is detected on backward computation'
                     raise RuntimeError(msg)


Test patch:
diff --git a/tests/chainer_tests/test_function.py b/tests/chainer_tests/test_function.py
--- a/tests/chainer_tests/test_function.py
+++ b/tests/chainer_tests/test_function.py
@@ -316,13 +316,17 @@ def test_forward_invalid(self):
             f(v)
 
 
-class TestFunctionDebug(unittest.TestCase):
+@testing.parameterize(
+    {'return_value': (numpy.array([float('nan')], numpy.float32),),
+     'valid': False},
+    {'return_value': (numpy.array([1], numpy.int32),), 'valid': True},
+)
+class TestFunctionForwardDebug(unittest.TestCase):
 
     def setUp(self):
         self.original_debug = chainer.is_debug()
         chainer.set_debug(True)
         self.one = numpy.array([1], numpy.float32)
-        self.nan = numpy.array([float('nan')], numpy.float32)
         self.f = chainer.Function()
 
     def tearDown(self):
@@ -330,37 +334,66 @@ def tearDown(self):
 
     def check_debug_forward(self, x_data):
         x = chainer.Variable(x_data)
-        with self.assertRaises(RuntimeError):
+        if self.valid:
+            # check if forward throws nothing
             self.f(x)
+        else:
+            with self.assertRaises(RuntimeError):
+                self.f(x)
 
     def test_debug_forward_cpu(self):
-        self.f.forward_cpu = mock.MagicMock(return_value=(self.nan,))
+        self.f.forward_cpu = mock.MagicMock(return_value=self.return_value)
         self.check_debug_forward(self.one)
 
     @attr.gpu
     def test_debug_forward_gpu(self):
-        self.f.forward_gpu = mock.MagicMock(
-            return_value=(cuda.to_gpu(self.nan),))
+        return_value = tuple(None if x is None else cuda.to_gpu(x)
+                             for x in self.return_value)
+        self.f.forward_gpu = mock.MagicMock(return_value=return_value)
         self.check_debug_forward(cuda.to_gpu(self.one))
 
-    def check_debug_backward(self, x_data):
-        x = chainer.Variable(x_data)
-        y = self.f(x)
-        with self.assertRaises(RuntimeError):
+
+@testing.parameterize(
+    {'return_value': (numpy.array([float('nan')], numpy.float32),),
+     'valid': False},
+    {'return_value': (None,), 'valid': True},
+)
+class TestFunctionBackwardDebug(unittest.TestCase):
+
+    def setUp(self):
+        self.original_debug = chainer.is_debug()
+        chainer.set_debug(True)
+        self.one = numpy.array([1], numpy.float32)
+        self.f = chainer.Function()
+
+    def tearDown(self):
+        chainer.set_debug(self.original_debug)
+
+    def check_debug_backward(self, *xs_data):
+        xs = [chainer.Variable(x) for x in xs_data]
+        y = self.f(*xs)
+        if self.valid:
+            # check if backard throws nothing
             y.backward()
+        else:
+            with self.assertRaises(RuntimeError):
+                y.backward()
 
     def test_debug_backward_cpu(self):
         self.f.forward_cpu = mock.MagicMock(return_value=(self.one,))
-        self.f.backward_cpu = mock.MagicMock(return_value=(self.nan,))
-        self.check_debug_backward(self.one)
+        self.f.backward_cpu = mock.MagicMock(return_value=self.return_value)
+        input_value = (self.one,) * len(self.return_value)
+        self.check_debug_backward(*input_value)
 
     @attr.gpu
     def test_debug_backward_gpu(self):
         self.f.forward_gpu = mock.MagicMock(
             return_value=(cuda.to_gpu(self.one),))
-        self.f.backward_gpu = mock.MagicMock(
-            return_value=(cuda.to_gpu(self.nan),))
-        self.check_debug_backward(cuda.to_gpu(self.one))
+        return_value = tuple(None if x is None else cuda.to_gpu(x)
+                             for x in self.return_value)
+        input_value = (cuda.to_gpu(self.one),) * len(self.return_value)
+        self.f.backward_gpu = mock.MagicMock(return_value=return_value)
+        self.check_debug_backward(*input_value)
 
 
 testing.run_module(__name__, __file__)