Repository: chainer/chainer

Issue:
Support ignore label in `softmax_cross_entropy`
I want to ignore parts of labels in `softmax_cross_entropy`, for example output labels of RNNs. 



Discussion:

Resolving patch:
diff --git a/chainer/functions/loss/softmax_cross_entropy.py b/chainer/functions/loss/softmax_cross_entropy.py
--- a/chainer/functions/loss/softmax_cross_entropy.py
+++ b/chainer/functions/loss/softmax_cross_entropy.py
@@ -11,6 +11,8 @@ class SoftmaxCrossEntropy(function.Function):
 
     """Softmax activation followed by a cross entropy loss."""
 
+    ignore_label = -1
+
     def __init__(self, use_cudnn=True, normalize=True):
         self.use_cudnn = use_cudnn
         self.normalize = normalize
@@ -34,14 +36,20 @@ def forward_cpu(self, inputs):
         yd = numpy.rollaxis(self.y, 1)
         yd = yd.reshape(len(yd), -1).T
 
-        p = yd[six.moves.range(t.size), t.flat]
+        p = yd[six.moves.range(t.size), numpy.maximum(t.flat, 0)]
         # deal with the case where the SoftmaxCrossEntropy is
         # unpickled from the old version
         if getattr(self, 'normalize', True):
-            count = x.size // x.shape[1]
+            count = (t != self.ignore_label).sum()
         else:
             count = x.shape[0]
-        y = numpy.log(p).sum(keepdims=True) * (-1.0 / count)
+        self.count = count
+
+        if count == 0:
+            return numpy.zeros((), dtype=x.dtype),
+
+        y = (numpy.log(p) * (t.flat != self.ignore_label)).sum(keepdims=True) \
+            * (-1.0 / count)
         return y.reshape(()),
 
     def forward_gpu(self, inputs):
@@ -49,24 +57,33 @@ def forward_gpu(self, inputs):
         x, t = inputs
         self.y, = softmax.Softmax(self.use_cudnn).forward((x,))
         if getattr(self, 'normalize', True):
-            count = x.size // x.shape[1]
+            count = float((t != self.ignore_label).sum())
         else:
-            count = x.shape[0]
+            count = t.shape[0]
+        self.count = count
+
+        if count == 0:
+            return cupy.zeros((), dtype=x.dtype),
+
         y = cupy.rollaxis(self.y, 1, self.y.ndim)
         ret = cuda.reduce(
             'S t, raw T y, int32 n_channel, T inv_count', 'T out',
-            'log(y[_j * n_channel + t])',
+            't == -1 ? 0 : log(y[_j * n_channel + t])',
             'a + b', 'out = a * inv_count', '0', 'crossent_fwd'
         )(t, y.reduced_view(), y.shape[-1], -1.0 / count)
         return ret,
 
     def backward_cpu(self, inputs, grad_outputs):
         x, t = inputs
+        if self.count == 0:
+            return numpy.zeros_like(x), None
+
         gloss = grad_outputs[0]
-        n_unit = x.size // (x.shape[0] * x.shape[1])
+        n_unit = t.size // t.shape[0]
         if self.y.ndim == 2:
             gx = self.y.copy()
-            gx[six.moves.xrange(len(t)), t] -= 1
+            gx[six.moves.xrange(len(t)), numpy.maximum(t, 0)] -= 1
+            gx *= (t != self.ignore_label).reshape((len(t), 1))
         else:
             # in the case where y.ndim is higher than 2,
             # we think that a current implementation is inefficient
@@ -74,30 +91,33 @@ def backward_cpu(self, inputs, grad_outputs):
             gx = self.y.copy().reshape(self.y.shape[0], self.y.shape[1], -1)
             fst_index = numpy.arange(t.size) // n_unit
             trd_index = numpy.arange(t.size) % n_unit
-            gx[fst_index, t.flat, trd_index] -= 1
+            gx[fst_index, numpy.maximum(t.flat, 0), trd_index] -= 1
+            gx *= (t != self.ignore_label).reshape((len(t), 1, -1))
             gx = gx.reshape(self.y.shape)
 
-        if getattr(self, 'normalize', True):
-            count = t.shape[0] * n_unit
-        else:
-            count = t.shape[0]
-        gx *= gloss / count
+        gx *= gloss / self.count
         return gx, None
 
     def backward_gpu(self, inputs, grad_outputs):
         cupy = cuda.cupy
         x, t = inputs
+        if self.count == 0:
+            return cupy.zeros_like(x), None
+
         gloss = grad_outputs[0]
-        n_unit = x.size // (x.shape[0] * x.shape[1])
-        if getattr(self, 'normalize', True):
-            count = x.shape[0] * n_unit
-        else:
-            count = x.shape[0]
-        coeff = cuda.cupy.divide(gloss, count, dtype=gloss.dtype)
+        n_unit = t.size // t.shape[0]
+        coeff = cuda.cupy.divide(gloss, self.count, dtype=gloss.dtype)
         gx = cuda.elementwise(
             'T y, S t, raw T coeff, S n_channel, S n_unit',
             'T gx',
-            'gx = coeff[0] * (y - (t == (i / n_unit % n_channel)))',
+            '''
+               const int c = (i / n_unit % n_channel);
+               if (t == -1) {
+                 gx = 0;
+               } else {
+                 gx = coeff[0] * (y - (c == t));
+               }
+            ''',
             'softmax_crossent_bwd')(
                 self.y, cupy.expand_dims(t, 1), coeff, x.shape[1], n_unit)
         return gx, None
@@ -115,6 +135,7 @@ def softmax_cross_entropy(x, t, use_cudnn=True, normalize=True):
             to 2, it computes a cross entropy of the replicated softmax if the
             number of dimensions is greater than 2.
         t (Variable): Variable holding an int32 vector of groundtruth labels.
+            If ``t[i] == -1``, correspondig ``x[i]`` is ignored.
         normalize (Variable): Variable holding a boolean value which
             determines the normalization constant. If true, this function
             normalizes the cross entropy loss across all instances. If else,


Test patch:
diff --git a/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py b/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py
--- a/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py
+++ b/tests/chainer_tests/functions_tests/loss_tests/test_softmax_cross_entropy.py
@@ -30,9 +30,17 @@ def check_forward(self, x_data, t_data, use_cudnn=True):
         # Compute expected value
         y = numpy.exp(self.x)
         loss_expect = 0.0
+        count = 0
         for i in six.moves.range(y.shape[0]):
+            if self.t[i] == -1:
+                continue
             loss_expect -= math.log(y[i, self.t[i]] / y[i].sum())
-        loss_expect /= y.shape[0]
+            count += 1
+
+        if count == 0:
+            loss_expect = 0.0
+        else:
+            loss_expect /= count
 
         self.assertAlmostEqual(loss_expect, loss_value, places=5)
 
@@ -96,11 +104,19 @@ def check_forward(self, x_data, t_data, use_cudnn=True):
         # Compute expected value
         y = numpy.exp(self.x)
         loss_expect = 0.0
+        count = 0
         for i in six.moves.range(y.shape[0]):
             for k in six.moves.range(y.shape[2]):
+                if self.t[i, k] == -1:
+                    continue
                 loss_expect -= math.log(
                     y[i, self.t[i, k], k] / y[i, :, k].sum())
-        loss_expect /= y.shape[0] * y.shape[2]
+                count += 1
+
+        if count == 0:
+            loss_expect = 0.0
+        else:
+            loss_expect /= count
 
         self.assertAlmostEqual(loss_expect, loss_value, places=4)
 
@@ -127,10 +143,59 @@ def check_forward(self, x_data, t_data, use_cudnn=True):
         for i in six.moves.range(y.shape[0]):
             for k in six.moves.range(y.shape[2]):
                 for l in six.moves.range(y.shape[3]):
+                    if self.t[i, k, l] == -1:
+                        continue
                     loss_expect -= math.log(
                         y[i, self.t[i, k, l], k, l] / y[i, :, k, l].sum())
         loss_expect /= y.shape[0]
 
         self.assertAlmostEqual(loss_expect, loss_value, places=4)
 
+
+class TestSoftmaxCrossEntropyWithIgnoreLabel(TestSoftmaxCrossEntropy):
+
+    def setUp(self):
+        super(TestSoftmaxCrossEntropyWithIgnoreLabel, self).setUp()
+        self.t[2] = -1
+
+
+class TestSoftmaxCrossEntropyIgnoreAll(TestSoftmaxCrossEntropy):
+
+    def setUp(self):
+        super(TestSoftmaxCrossEntropyIgnoreAll, self).setUp()
+        self.t[:] = -1
+
+
+class TestReplicatedSoftmaxCrossEntropy1IgnoreLabel(
+        TestReplicatedSoftmaxCrossEntropy1):
+
+    def setUp(self):
+        super(TestReplicatedSoftmaxCrossEntropy1IgnoreLabel, self).setUp()
+        self.t[0, 1] = -1
+
+
+class TestReplicatedSoftmaxCrossEntropy2IgnoreLabel(
+        TestReplicatedSoftmaxCrossEntropy2):
+
+    def setUp(self):
+        super(TestReplicatedSoftmaxCrossEntropy2IgnoreLabel, self).setUp()
+        self.t[0, 1, 2] = -1
+
+
+class TestReplicatedSoftmaxCrossEntropy1IgnoreAll(
+        TestReplicatedSoftmaxCrossEntropy1):
+
+    def setUp(self):
+        super(TestReplicatedSoftmaxCrossEntropy1IgnoreAll, self).setUp()
+        self.t[:] = -1
+
+
+class TestReplicatedSoftmaxCrossEntropy2IgnoreAll(
+        TestReplicatedSoftmaxCrossEntropy2):
+
+    def setUp(self):
+        super(TestReplicatedSoftmaxCrossEntropy2IgnoreAll, self).setUp()
+        self.t[:] = -1
+
+
 testing.run_module(__name__, __file__)