Repository: chainer/chainer

Issue:
Assert that entries to softmax are appropriate
No single entry in the input array of the softmax should exceed that shape of the input array.

So for example,  if you call `F.softmax(y)` then `assert y.max() < y.shape[1]` should be True. This does introduce some overhead, so perhaps is worth debating -- but I just spent a few hours debugging such a bug so I'm in favor :)



Discussion:
That would be very helpful for debugging. In order to keep the maximum speed, we might have to think about introducing _debug mode_ to diagnose such semantic errors with additional overheads.


Resolving patch:
diff --git a/chainer/functions/array/select_item.py b/chainer/functions/array/select_item.py
--- a/chainer/functions/array/select_item.py
+++ b/chainer/functions/array/select_item.py
@@ -1,6 +1,7 @@
 import numpy
 import six
 
+import chainer
 from chainer import cuda
 from chainer import function
 from chainer.utils import type_check
@@ -22,21 +23,27 @@ def check_type_forward(self, in_types):
             x_type.shape[0] == t_type.shape[0],
         )
 
-    def forward_cpu(self, inputs):
+    def forward(self, inputs):
         x, t = inputs
-        # This code is equivalent to `t.choose(x.T)`, but `numpy.choose`
-        # does not work when `x.shape[1] > 32`.
-        return x[six.moves.range(t.size), t],
-
-    def forward_gpu(self, inputs):
-        x, t = inputs
-        y = cuda.elementwise(
-            'S t, raw T x',
-            'T y',
-            'int ind[] = {i, t}; y = x[ind];',
-            'getitem_fwd'
-        )(t, x)
-        return y,
+        if chainer.is_debug():
+            if not ((0 <= t).all() and
+                    (t < x.shape[1]).all()):
+                msg = 'Each label `t` need to satisfty `0 <= t < x.shape[1]`'
+                raise ValueError(msg)
+
+        xp = cuda.get_array_module(x)
+        if xp is numpy:
+            # This code is equivalent to `t.choose(x.T)`, but `numpy.choose`
+            # does not work when `x.shape[1] > 32`.
+            return x[six.moves.range(t.size), t],
+        else:
+            y = cuda.elementwise(
+                'S t, raw T x',
+                'T y',
+                'int ind[] = {i, t}; y = x[ind];',
+                'getitem_fwd'
+            )(t, x)
+            return y,
 
     def backward_cpu(self, inputs, grad_outputs):
         x, t = inputs
diff --git a/chainer/functions/connection/embed_id.py b/chainer/functions/connection/embed_id.py
--- a/chainer/functions/connection/embed_id.py
+++ b/chainer/functions/connection/embed_id.py
@@ -1,6 +1,7 @@
 import numpy
 import six
 
+import chainer
 from chainer import cuda
 from chainer import function
 from chainer.utils import type_check
@@ -22,6 +23,12 @@ def check_type_forward(self, in_types):
 
     def forward(self, inputs):
         x, W = inputs
+        if chainer.is_debug():
+            if not ((0 <= x).all() and
+                    (x < len(W)).all()):
+                msg = 'Each `x` value need to satisfty `0 <= x < len(W)`'
+                raise ValueError(msg)
+
         return W.take(x, axis=0),
 
     def backward(self, inputs, grad_outputs):
diff --git a/chainer/functions/loss/softmax_cross_entropy.py b/chainer/functions/loss/softmax_cross_entropy.py
--- a/chainer/functions/loss/softmax_cross_entropy.py
+++ b/chainer/functions/loss/softmax_cross_entropy.py
@@ -1,6 +1,7 @@
 import numpy
 import six
 
+import chainer
 from chainer import cuda
 from chainer import function
 from chainer.utils import type_check
@@ -67,8 +68,19 @@ def check_type_forward(self, in_types):
             x_type.shape[2:] == t_type.shape[1:],
         )
 
+    def _check_input_values(self, x, t):
+        if not (((0 <= t) &
+                 (t < x.shape[1])) |
+                (t == self.ignore_label)).all():
+            msg = ('Each label `t` need to satisfty '
+                   '`0 <= t < x.shape[1] or t == %d`' % self.ignore_label)
+            raise ValueError(msg)
+
     def forward_cpu(self, inputs):
         x, t = inputs
+        if chainer.is_debug():
+            self._check_input_values(x, t)
+
         log_y = softmax_log(x, False)
         self.y = numpy.exp(log_y)
         log_yd = numpy.rollaxis(log_y, 1)
@@ -90,6 +102,9 @@ def forward_cpu(self, inputs):
     def forward_gpu(self, inputs):
         cupy = cuda.cupy
         x, t = inputs
+        if chainer.is_debug():
+            self._check_input_values(x, t)
+
         log_y = softmax_log(x, self.use_cudnn)
         self.y = cupy.exp(log_y)
         if getattr(self, 'normalize', True):


Test patch:
diff --git a/tests/chainer_tests/functions_tests/array_tests/test_select_item.py b/tests/chainer_tests/functions_tests/array_tests/test_select_item.py
--- a/tests/chainer_tests/functions_tests/array_tests/test_select_item.py
+++ b/tests/chainer_tests/functions_tests/array_tests/test_select_item.py
@@ -56,4 +56,39 @@ def test_backward_gpu(self):
                             cuda.to_gpu(self.gy_data))
 
 
+@testing.parameterize(
+    {'t_value': -1, 'valid': False},
+    {'t_value': 3,  'valid': False},
+    {'t_value': 0,  'valid': True},
+)
+class TestSelectItemValueCheck(unittest.TestCase):
+
+    def setUp(self):
+        self.x = numpy.empty((1, 2), dtype=numpy.float32)
+        self.t = numpy.array([self.t_value], dtype=numpy.int32)
+        self.original_debug = chainer.is_debug()
+        chainer.set_debug(True)
+
+    def tearDown(self):
+        chainer.set_debug(self.original_debug)
+
+    def check_value_check(self, x_data, t_data):
+        x = chainer.Variable(x_data)
+        t = chainer.Variable(t_data)
+
+        if self.valid:
+            # Check if it throws nothing
+            functions.select_item(x, t)
+        else:
+            with self.assertRaises(ValueError):
+                functions.select_item(x, t)
+
+    def test_value_check_cpu(self):
+        self.check_value_check(self.x, self.t)
+
+    @attr.gpu
+    def test_value_check_gpu(self):
+        self.check_value_check(self.x, self.t)
+
+
 testing.run_module(__name__, __file__)
diff --git a/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py b/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py
--- a/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py
+++ b/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py
@@ -202,6 +202,46 @@ def setUp(self):
         self.t[:] = -1
 
 
+@testing.parameterize(
+    {'t_value': -2, 'valid': False},
+    {'t_value': 3,  'valid': False},
+    {'t_value': -1, 'valid': True},  # -1 is ignore_label
+)
+class TestSoftmaxCrossEntropyValueCheck(unittest.TestCase):
+
+    def setUp(self):
+        self.x = numpy.empty((2, 2), dtype=numpy.float32)
+        # `0` is required to avoid NaN
+        self.t = numpy.array([self.t_value, 0], dtype=numpy.int32)
+        self.original_debug = chainer.is_debug()
+        chainer.set_debug(True)
+
+    def tearDown(self):
+        chainer.set_debug(self.original_debug)
+
+    def check_value_check(self, x_data, t_data, use_cudnn):
+        x = chainer.Variable(x_data)
+        t = chainer.Variable(t_data)
+
+        if self.valid:
+            # Check if it throws nothing
+            functions.softmax_cross_entropy(x, t, use_cudnn)
+        else:
+            with self.assertRaises(ValueError):
+                functions.softmax_cross_entropy(x, t, use_cudnn)
+
+    def test_value_check_cpu(self):
+        self.check_value_check(self.x, self.t, False)
+
+    @attr.gpu
+    def test_value_check_gpu(self):
+        self.check_value_check(self.x, self.t, False)
+
+    @attr.cudnn
+    def test_value_check_gpu_cudnn(self):
+        self.check_value_check(cuda.to_gpu(self.x), cuda.to_gpu(self.t), True)
+
+
 @testing.parameterize(
     {'use_cudnn': True},
     {'use_cudnn': False},
diff --git a/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py b/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py
--- a/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py
+++ b/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py
@@ -62,4 +62,38 @@ def test_backward_gpu(self):
         self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))
 
 
+@testing.parameterize(
+    {'t_value': -1, 'valid': False},
+    {'t_value': 3,  'valid': False},
+    {'t_value': 0,  'valid': True},
+)
+class TestEmbedIDValueCheck(unittest.TestCase):
+
+    def setUp(self):
+        self.link = links.EmbedID(2, 2)
+        self.t = numpy.array([self.t_value], dtype=numpy.int32)
+        self.original_debug = chainer.is_debug()
+        chainer.set_debug(True)
+
+    def tearDown(self):
+        chainer.set_debug(self.original_debug)
+
+    def check_value_check(self, t_data):
+        t = chainer.Variable(t_data)
+
+        if self.valid:
+            # Check if it throws nothing
+            self.link(t)
+        else:
+            with self.assertRaises(ValueError):
+                self.link(t)
+
+    def test_value_check_cpu(self):
+        self.check_value_check(self.t)
+
+    @attr.gpu
+    def test_value_check_gpu(self):
+        self.check_value_check(self.t)
+
+
 testing.run_module(__name__, __file__)
diff --git a/tests/chainer_tests/test_function.py b/tests/chainer_tests/test_function.py
--- a/tests/chainer_tests/test_function.py
+++ b/tests/chainer_tests/test_function.py
@@ -319,13 +319,14 @@ def test_forward_invalid(self):
 class TestFunctionDebug(unittest.TestCase):
 
     def setUp(self):
+        self.original_debug = chainer.is_debug()
         chainer.set_debug(True)
         self.one = numpy.array([1], numpy.float32)
         self.nan = numpy.array([float('nan')], numpy.float32)
         self.f = chainer.Function()
 
     def tearDown(self):
-        chainer.set_debug(False)
+        chainer.set_debug(self.original_debug)
 
     def check_debug_forward(self, x_data):
         x = chainer.Variable(x_data)