Repository: chainer/chainer

Issue:
Making the CTC function to be the same as Graves 06
In Graves 06, eq.(6) tells that the first line should be taken if l'\_s = b or l'\_{s-2} = l'\_s.
However, in ConnectionistTemporalClassification class of the chainer implementation, only the l'\_s = b condition seems to be implemented.
This makes the CTC probability larger when the same label is repeated, then the CTC loss goes smaller, and the network tend to give repeated labels excessively.  We observed slowness of the training process too.
(Supposing that the repeated labels are 'aa', the current implementation allows both 'aaa' and  'a_a' output, however, the Graves 06's paper supposes that only 'a_a' output should be allowed.)
The blog post https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/ , on which your code is based, seems to have the same problem.


Discussion:

Resolving patch:
diff --git a/chainer/functions/loss/ctc.py b/chainer/functions/loss/ctc.py
--- a/chainer/functions/loss/ctc.py
+++ b/chainer/functions/loss/ctc.py
@@ -87,16 +87,24 @@ def log_matrix(self, x, xp):
             res = create_recurrence_relation(x, self.zero_padding)
         return res.astype(numpy.float32)
 
-    def recurrence_relation(self, path_length, max_length, dtype, xp):
+    def recurrence_relation(self, label, path_length, max_length, dtype, xp):
         """Transition in forword and backword algorithms is represented as matrix.
 
         See also
         https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
         """
-        rr = (xp.eye(max_length, dtype=dtype) +
-              xp.eye(max_length, k=1, dtype=dtype) +
-              xp.eye(max_length, k=2, dtype=dtype) *
-              (xp.arange(max_length, dtype=dtype) % dtype(2)))
+        batch, lab = label.shape
+        repeat_mask = xp.ones((batch, lab * 2 + 1))
+        repeat_mask[:, 1::2] = (label !=
+                                xp.take(label, xp.arange(-1, lab - 1)
+                                        % lab + xp.arange(0, batch * lab,
+                                                          lab)[:, None]))
+        repeat_mask[:, 1] = 1
+        rr = (xp.eye(max_length, dtype=dtype)[None, :] +
+              xp.eye(max_length, k=1, dtype=dtype)[None, :] +
+              (xp.eye(max_length, k=2, dtype=dtype) *
+               (xp.arange(max_length, dtype=dtype) % dtype(2))[None, :]
+               * repeat_mask[:, None]))
         return self.log_matrix(
             rr * (path_length[:, None] > xp.arange(max_length))[..., None], xp)
 
@@ -145,7 +153,8 @@ def label_probability(self, label_size, path, path_length,
                                           path.shape[1], ret[i])
         return ret
 
-    def calc_trans(self, path, yseq, xp):
+    def calc_trans(self, yseq, input_length,
+                   label, label_length, path, path_length, xp):
         forward_prob = self.log_matrix(
             xp.eye(path.shape[1], dtype='f')[0], xp)[None, :]
         backward_prob = forward_prob
@@ -155,7 +164,7 @@ def calc_trans(self, path, yseq, xp):
         # prob[i] := forward[i] + backward[-i-1]
         index = offset + path
         frr = self.recurrence_relation(
-            self.path_length, path.shape[1], numpy.float32, xp)
+            label, path_length, path.shape[1], numpy.float32, xp)
         prob = xp.empty(
             (len(yseq),) + index.shape, dtype=forward_prob.dtype)
         # forward computation.
@@ -164,21 +173,21 @@ def calc_trans(self, path, yseq, xp):
             forward_prob = xp.take(y, index) + _log_dot(
                 forward_prob[:, None, :], frr, xp)
             prob[i] = forward_prob
-        r_index = offset + _move_label_to_back(path, self.path_length, xp)
+        r_index = offset + _move_label_to_back(path, path_length, xp)
 
         # rotate yseq with path_length
-        yseq_inv = _move_inputs(yseq, self.input_length, xp)[::-1]
+        yseq_inv = _move_inputs(yseq, input_length, xp)[::-1]
         brr = self.recurrence_relation(
-            self.path_length, path.shape[1], numpy.float32, xp)
-
+            _move_label_to_back(label, label_length, xp),
+            path_length, path.shape[1], numpy.float32, xp)
         # move to back.
-        prob = _move_inputs(prob, self.input_length, xp)
+        prob = _move_inputs(prob, input_length, xp)
 
         # backward computation.
         ps1 = path.shape[1]
         backward_prob_index = (
             xp.arange(0, path.size, ps1, dtype=numpy.int32)[:, None] +
-            (xp.arange(ps1) - self.path_length[:, None]) % ps1)
+            (xp.arange(ps1) - path_length[:, None]) % ps1)
         for i, y_inv in enumerate(yseq_inv):
             # calc backward probability
             backward_prob = _log_dot(backward_prob[:, None, :], brr, xp)
@@ -213,7 +222,9 @@ def forward(self, inputs):
         self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
         log_yseq = self.log_matrix(self.yseq, xp)
         self.path = _label_to_path(t, self.blank_symbol, xp)
-        self.prob_trans = self.calc_trans(self.path, log_yseq, xp)
+        self.prob_trans = self.calc_trans(
+            log_yseq, self.input_length, t,
+            label_length, self.path, self.path_length, xp)
 
         loss = utils.force_array(xp.sum(
             _logsumexp(self.prob_trans[0], xp, axis=1)))


Test patch:
diff --git a/tests/chainer_tests/functions_tests/loss_tests/test_ctc.py b/tests/chainer_tests/functions_tests/loss_tests/test_ctc.py
--- a/tests/chainer_tests/functions_tests/loss_tests/test_ctc.py
+++ b/tests/chainer_tests/functions_tests/loss_tests/test_ctc.py
@@ -131,6 +131,16 @@ def setUp(self):
         self.l_length[...] = 1
 
 
+class TestCTCWithRepeatedLabel(TestCTC):
+
+    def setUp(self):
+        super(TestCTCWithRepeatedLabel, self).setUp()
+        self.t = numpy.array([[0, 1, 1], [0, 1, 0]]).astype(numpy.int32)
+        self.l = numpy.array([[2, 0, 2, 1, 2, 1, 2],
+                              [2, 0, 2, 1, 2, 0, 2]]).astype(numpy.int32)
+        self.l_length = numpy.full((len(self.t),), len(self.t[0]), dtype='i')
+
+
 class TestCTCBlankSymbol(TestCTC):
 
     def setUp(self):