Repository: chainer/chainer

Issue:
Bad gradients pass gradient check
Even when the computed gradients are wrong, the gradient check tests seem to pass.
This started occurring since merge https://github.com/chainer/chainer/commit/85372a2d909f1649a83aab4fd3c4fab0dfa6e732

It is easily reproducible, by simply altering the outputs of some function and then running the corresponding test, noticing that the test still passes.


Discussion:

Resolving patch:
diff --git a/chainer/gradient_check.py b/chainer/gradient_check.py
--- a/chainer/gradient_check.py
+++ b/chainer/gradient_check.py
@@ -282,7 +282,7 @@ def check_backward(func, x_data, y_grad, params=(),
                 raise RuntimeError(
                     'gradients of some arguments are not calculated')
 
-    if len(xs) - len(no_grads) + len(params) == 0:
+    if len(xs) - no_grads.count(True) + len(params) == 0:
         # When there is no float variables, we need not to check gradient
         # values
         return
@@ -302,8 +302,10 @@ def check_backward(func, x_data, y_grad, params=(),
     directions = [xp.random.normal(size=x.shape) for x in variables]
     # Use unit vector
     norm = math.sqrt(sum([xp.square(d).sum() for d in directions]))
-    scale = 1. / norm
-    directions = [d * scale for d in directions]
+    if norm != 0:
+        # norm could be zero if input arrays are 0-sized.
+        scale = 1. / norm
+        directions = [d * scale for d in directions]
 
     delta = xp.array(0., 'd')
 


Test patch:
diff --git a/tests/chainer_tests/test_gradient_check.py b/tests/chainer_tests/test_gradient_check.py
--- a/tests/chainer_tests/test_gradient_check.py
+++ b/tests/chainer_tests/test_gradient_check.py
@@ -370,6 +370,96 @@ def f(x, y):
         gradient_check.check_backward(f, (x1, x2), g1, no_grads=[False, True])
 
 
+class TestCheckBackwardFailure(unittest.TestCase):
+
+    def _broken_func_1(self):
+        class Broken(chainer.Function):
+            def forward(self, inputs):
+                x, = inputs
+                return (x * x),
+
+            def backward(self, inputs, grad_outputs):
+                x, = inputs
+                gy, = grad_outputs
+                return 3 * x * gy,
+
+        return Broken()
+
+    def _broken_func_2(self):
+        class Broken(chainer.FunctionNode):
+            def forward(self, inputs):
+                x, = inputs
+                self.retain_inputs((0,))
+                return (x * x),
+
+            def backward(self, indexes, grad_outputs):
+                x, = self.get_retained_inputs()
+                gy, = grad_outputs
+                return 3 * x * gy,
+
+        return Broken()
+
+    def _broken_func_3(self):
+        class Broken(chainer.FunctionNode):
+            def forward(self, inputs):
+                x, = inputs
+                self.retain_inputs((0,))
+                return (x * x),
+
+            def backward(self, indexes, grad_outputs):
+                x, = self.get_retained_inputs()
+                gy, = grad_outputs
+                gx1 = 2 * x * gy
+                gx2 = 3 * x * gy
+                return (gx1, gx2)
+
+        return Broken()
+
+    def test_fail_function(self):
+        # Invalid backward (chainer.Function)
+        x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
+        gy = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
+
+        def f(x):
+            return self._broken_func_1()(x)
+
+        with self.assertRaises(AssertionError):
+            gradient_check.check_backward(f, x, gy)
+
+    def test_fail_function_node(self):
+        # Invalid backward (chainer.FunctionNode)
+        x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
+        gy = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
+
+        def f(x):
+            return self._broken_func_2().apply((x,))
+
+        with self.assertRaises(AssertionError):
+            gradient_check.check_backward(f, x, gy)
+
+    def test_fail_invalid_number_of_gradients(self):
+        # Invalid number of gradients
+        x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
+        gy = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
+
+        def f(x):
+            return self._broken_func_3().apply((x,))
+
+        with self.assertRaises(ValueError):
+            gradient_check.check_backward(f, x, gy)
+
+    def test_fail_invalid_number_of_gradients_0_size(self):
+        # Invalid number of gradients (0-sized input)
+        x = numpy.random.uniform(-1, 1, (2, 0)).astype(numpy.float32)
+        gy = numpy.random.uniform(-1, 1, (2, 0)).astype(numpy.float32)
+
+        def f(x):
+            return self._broken_func_3().apply((x,))
+
+        with self.assertRaises(ValueError):
+            gradient_check.check_backward(f, x, gy)
+
+
 class NewIdent(chainer.FunctionNode):
 
     def forward(self, inputs):