Repository: chainer/chainer

Issue:
Inconsistency of type checking between SoftmaxCrossEntropy and Accuracy
We often evaluate training procedure with both `softmax_cross_entropy` and `accuracy`, as default `chainer.links.Classifier` does. But there are cases `softmax_cross_entorpy(x, t)` is accepted but `accuracy(x, t)` raises error because of its type checking (and vise versa). Replicated softmax is one of a such examples (e.g. `x.shape = (B, C, N)` and `t.shape = (B, N)`). We need to reshape them manually in such a case.

It is convenient for users that shape checking of the two be more consistent.



Discussion:

Resolving patch:
diff --git a/chainer/functions/evaluation/accuracy.py b/chainer/functions/evaluation/accuracy.py
--- a/chainer/functions/evaluation/accuracy.py
+++ b/chainer/functions/evaluation/accuracy.py
@@ -1,4 +1,5 @@
 import numpy
+import six
 
 from chainer import cuda
 from chainer import function
@@ -16,18 +17,21 @@ def check_type_forward(self, in_types):
 
         type_check.expect(
             x_type.dtype.kind == 'f',
-            x_type.ndim >= 2,
-            t_type.dtype == numpy.int32,
-            t_type.ndim == 1,
-            t_type.shape[0] == x_type.shape[0],
+            t_type.dtype == numpy.int32
         )
-        for i in range(2, x_type.ndim.eval()):
+
+        t_ndim = t_type.ndim.eval()
+        type_check.expect(
+            x_type.ndim >= t_type.ndim,
+            x_type.shape[0] == t_type.shape[0],
+            x_type.shape[2: t_ndim + 1] == t_type.shape[1:]
+        )
+        for i in six.moves.range(t_ndim + 1, x_type.ndim.eval()):
             type_check.expect(x_type.shape[i] == 1)
 
     def forward(self, inputs):
         xp = cuda.get_array_module(*inputs)
         y, t = inputs
-        y = y.reshape(len(y), -1)  # flatten
 
         if self.ignore_label is not None:
             mask = (t == self.ignore_label)
@@ -37,16 +41,17 @@ def forward(self, inputs):
             # TODO(henry0312)
             #   If cupy.where returns indexes, we could make the code better.
             #   Also, we would need Advanced Indexing.
-            pred = xp.where(mask, self.ignore_label, y.argmax(axis=1))
+            pred = xp.where(mask, self.ignore_label,
+                            y.argmax(axis=1).reshape(t.shape))
             count = (pred == t).sum() - ignore_cnt
-            total = len(t) - ignore_cnt
+            total = t.size - ignore_cnt
 
             if total == 0:
                 return xp.asarray(0.0, dtype=y.dtype),
             else:
                 return xp.asarray(float(count) / total, dtype=y.dtype),
         else:
-            pred = y.argmax(axis=1)
+            pred = y.argmax(axis=1).reshape(t.shape)
             return xp.asarray((pred == t).mean(dtype=y.dtype)),
 
 


Test patch:
diff --git a/tests/chainer_tests/functions_tests/evaluation_tests/test_accuracy.py b/tests/chainer_tests/functions_tests/evaluation_tests/test_accuracy.py
--- a/tests/chainer_tests/functions_tests/evaluation_tests/test_accuracy.py
+++ b/tests/chainer_tests/functions_tests/evaluation_tests/test_accuracy.py
@@ -9,24 +9,60 @@
 from chainer import testing
 from chainer.testing import attr
 from chainer.testing import condition
+from chainer.utils import type_check
 
 
-@testing.parameterize(*testing.product({
-    'shape': [(10, 3), (20, 3)],
-    't_data': ['random', 'zero'],
-    'ignore_label': [None, 0],
-    'dtype': [numpy.float16, numpy.float32, numpy.float64],
-}))
+def accuracy(x, t, ignore_label):
+    x_ = numpy.rollaxis(x, 1, x.ndim).reshape(t.size, -1)
+    t_ = t.ravel()
+
+    if ignore_label is not None:
+        count = 0
+        for i in six.moves.range(t_.size):
+            pred = x_[i].argmax()
+            if t_[i] != ignore_label and pred == t_[i]:
+                count += 1
+        total = (t_ != ignore_label).sum()
+    else:
+        count = 0
+        for i in six.moves.range(t_.size):
+            pred = x_[i].argmax()
+            if pred == t_[i]:
+                count += 1
+        total = t_.size
+
+    if total == 0:
+        return 0.0
+    else:
+        return float(count) / total
+
+
+@testing.parameterize(
+    *testing.product_dict(
+        [{'x_shape': (10, 3), 't_shape': (10,)},
+         {'x_shape': (10, 3, 1), 't_shape': (10,)},
+         {'x_shape': (10, 3, 1, 1), 't_shape': (10,)},
+         {'x_shape': (10, 3, 5), 't_shape': (10, 5)},
+         {'x_shape': (10, 3, 5, 4), 't_shape': (10, 5, 4)},
+         {'x_shape': (10, 3, 5, 4, 1), 't_shape': (10, 5, 4)},
+         {'x_shape': (10, 3, 5, 4, 1, 1), 't_shape': (10, 5, 4)}],
+        [{'ignore_label': None, 't_data': 'randint'},
+         {'ignore_label': 0, 't_data': 'randint'},
+         {'ignore_label': 0, 't_data': 'zero'}],
+        [{'dtype': numpy.float16},
+         {'dtype': numpy.float32},
+         {'dtype': numpy.float64}]
+    )
+)
 class TestAccuracy(unittest.TestCase):
 
     def setUp(self):
-        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
-        if self.t_data == 'random':
+        self.x = numpy.random.uniform(-1, 1, self.x_shape).astype(self.dtype)
+        if self.t_data == 'randint':
             self.t = numpy.random.randint(
-                3, size=self.shape[:1]).astype(numpy.int32)
+                3, size=self.t_shape).astype(numpy.int32)
         elif self.t_data == 'zero':
-            self.t = numpy.zeros(self.shape[:1], dtype=numpy.int32)
-
+            self.t = numpy.zeros(self.t_shape).astype(numpy.int32)
         self.check_forward_options = {}
         if self.dtype == numpy.float16:
             self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
@@ -38,27 +74,9 @@ def check_forward(self, x_data, t_data):
         self.assertEqual(y.data.dtype, self.dtype)
         self.assertEqual((), y.data.shape)
 
-        if self.ignore_label is not None:
-            count = 0
-            for i in six.moves.range(self.t.size):
-                pred = self.x[i].argmax()
-                if self.t[i] != self.ignore_label and pred == self.t[i]:
-                    count += 1
-                total = (self.t != self.ignore_label).sum()
-        else:
-            count = 0
-            for i in six.moves.range(self.t.size):
-                pred = self.x[i].argmax()
-                if pred == self.t[i]:
-                    count += 1
-                total = self.t.size
-
-        if total == 0:
-            expected = 0.0
-        else:
-            expected = float(count) / total
-        gradient_check.assert_allclose(
-            expected, cuda.to_cpu(y.data), **self.check_forward_options)
+        expected = accuracy(self.x, self.t, self.ignore_label)
+        gradient_check.assert_allclose(expected, cuda.to_cpu(y.data),
+                                       **self.check_forward_options)
 
     @condition.retry(3)
     def test_forward_cpu(self):
@@ -70,4 +88,33 @@ def test_forward_gpu(self):
         self.check_forward(cuda.to_gpu(self.x), cuda.to_gpu(self.t))
 
 
+@testing.parameterize(
+    {'x_shape': (10, 3), 't_shape': (4,)},
+    {'x_shape': (10, 3, 2), 't_shape': (10,)},
+    {'x_shape': (10, 3, 1, 2), 't_shape': (10,)},
+    {'x_shape': (10, 3, 4), 't_shape': (10, 5)},
+    {'x_shape': (10, 3, 5, 2), 't_shape': (10, 5)},
+    {'x_shape': (10, 3, 5, 1, 2), 't_shape': (10, 5)},
+)
+class TestInvalidShape(unittest.TestCase):
+
+    def setUp(self):
+        self.x = numpy.random.uniform(-1, 1,
+                                      self.x_shape).astype(numpy.float32)
+        self.t = numpy.random.randint(3, size=self.t_shape).astype(numpy.int32)
+
+    def check_invalid_shape(self, xp):
+        x = chainer.Variable(xp.asarray(self.x))
+        t = chainer.Variable(xp.asarray(self.t))
+        with self.assertRaises(type_check.InvalidType):
+            chainer.functions.accuracy(x, t)
+
+    def test_invalid_shape_cpu(self):
+        self.check_invalid_shape(numpy)
+
+    @attr.gpu
+    def test_invalid_shape_gpu(self):
+        self.check_invalid_shape(cuda.cupy)
+
+
 testing.run_module(__name__, __file__)