Repository: chainer/chainer

Issue:
Ignore labels for EmbedID
Loss functions supports ignore labels, but `EmbedID` does not. When a user set label variable = -1, it causes out-of-bounds problem in EmbedID. It is common problem in language modeling because input variables and output variables for LM are usually shared.



Discussion:
Also true for `NegativeSampling`. The loss functions `sigmoid_cross_entropy` and `softmax_cross_entropy` support ignore labels, but negative sampling does not.

It's a good suggestion!


Resolving patch:
diff --git a/chainer/functions/connection/embed_id.py b/chainer/functions/connection/embed_id.py
--- a/chainer/functions/connection/embed_id.py
+++ b/chainer/functions/connection/embed_id.py
@@ -9,6 +9,9 @@
 
 class EmbedIDFunction(function.Function):
 
+    def __init__(self, ignore_label=None):
+        self.ignore_label = ignore_label
+
     def check_type_forward(self, in_types):
         type_check.expect(in_types.size() == 2)
         x_type, w_type = in_types
@@ -23,12 +26,19 @@ def check_type_forward(self, in_types):
 
     def forward(self, inputs):
         x, W = inputs
+
         if chainer.is_debug():
             if not ((0 <= x).all() and
                     (x < len(W)).all()):
                 msg = 'Each `x` value need to satisfty `0 <= x < len(W)`'
                 raise ValueError(msg)
 
+        if self.ignore_label is not None:
+            xp = cuda.get_array_module(*inputs)
+            mask = (x == self.ignore_label)
+            return xp.where(
+                mask[..., None], 0, W.take(xp.where(mask, 0, x), axis=0)),
+
         return W.take(x, axis=0),
 
     def backward(self, inputs, grad_outputs):
@@ -42,17 +52,32 @@ def backward(self, inputs, grad_outputs):
             # too slow.
             for ix, igy in six.moves.zip(x.ravel(),
                                          gy.reshape(x.size, -1)):
+                if ix == self.ignore_label:
+                    continue
                 gW[ix] += igy
         else:
-            cuda.elementwise(
-                'T gy, int32 x, int32 n_out', 'raw T gW',
-                'int w_ind[] = {x, i % n_out}; atomicAdd(&gW[w_ind], gy)',
-                'embed_id_bwd')(
-                    gy, xp.expand_dims(x, -1), gW.shape[1], gW)
+            if self.ignore_label is None:
+                cuda.elementwise(
+                    'T gy, int32 x, int32 n_out', 'raw T gW',
+                    'int w_ind[] = {x, i % n_out}; atomicAdd(&gW[w_ind], gy)',
+                    'embed_id_bwd')(
+                        gy, xp.expand_dims(x, -1), gW.shape[1], gW)
+            else:
+                cuda.elementwise(
+                    'T gy, int32 x, int32 n_out, int32 ignore', 'raw T gW',
+                    '''
+                    if (x != ignore) {
+                      int w_ind[] = {x, i % n_out};
+                      atomicAdd(&gW[w_ind], gy);
+                    }
+                    ''',
+                    'embed_id_bwd_ignore_label')(
+                        gy, xp.expand_dims(x, -1), gW.shape[1],
+                        self.ignore_label, gW)
         return None, gW
 
 
-def embed_id(x, W):
+def embed_id(x, W, ignore_label=None):
     """Efficient linear function for one-hot input.
 
     This function implements so called *word embedding*. It takes two
@@ -67,6 +92,8 @@ def embed_id(x, W):
         x (~chainer.Variable): Batch vectors of IDs.
         W (~chainer.Variable): Representation of each ID (a.k.a.
             word embeddings).
+        ignore_label (int or None): If ``ignore_label`` is an int value,
+            ``i``-th column of return value is filled with ``0``.
 
     Returns:
         ~chainer.Variable: Output variable.
@@ -74,4 +101,4 @@ def embed_id(x, W):
     .. seealso:: :class:`EmbedID`
 
     """
-    return EmbedIDFunction()(x, W)
+    return EmbedIDFunction(ignore_label=ignore_label)(x, W)
diff --git a/chainer/links/connection/embed_id.py b/chainer/links/connection/embed_id.py
--- a/chainer/links/connection/embed_id.py
+++ b/chainer/links/connection/embed_id.py
@@ -15,6 +15,8 @@ class EmbedID(link.Link):
         in_size (int): Number of different identifiers (a.k.a. vocabulary
             size).
         out_size (int): Size of embedding vector.
+        ignore_label (int or None): If ``ignore_label`` is an int value,
+            ``i``-th column of return value is filled with ``0``.
 
     .. seealso:: :func:`chainer.functions.embed_id`
 
@@ -22,9 +24,10 @@ class EmbedID(link.Link):
         W (~chainer.Variable): Embedding parameter matrix.
 
     """
-    def __init__(self, in_size, out_size):
+    def __init__(self, in_size, out_size, ignore_label=None):
         super(EmbedID, self).__init__(W=(in_size, out_size))
         self.W.data[...] = numpy.random.randn(in_size, out_size)
+        self.ignore_label = ignore_label
 
     def __call__(self, x):
         """Extracts the word embedding of given IDs.
@@ -36,4 +39,4 @@ def __call__(self, x):
             ~chainer.Variable: Batch of corresponding embeddings.
 
         """
-        return embed_id.embed_id(x, self.W)
+        return embed_id.embed_id(x, self.W, ignore_label=self.ignore_label)


Test patch:
diff --git a/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py b/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py
--- a/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py
+++ b/tests/chainer_tests/links_tests/connection_tests/test_embed_id.py
@@ -12,13 +12,16 @@
 
 
 @testing.parameterize(
-    {'x_data': [0, 1, 0]},
-    {'x_data': [[0, 1, 0], [1, 0, 1]]},
+    {'x_data': [0, 1, 0], 'ignore_label': None},
+    {'x_data': [[0, 1, 0], [1, 0, 1]], 'ignore_label': None},
+    {'x_data': [0, 1, -1], 'ignore_label': -1},
+    {'x_data': [[0, 1, -1], [-1, 0, 1]], 'ignore_label': -1},
 )
 class TestEmbedID(unittest.TestCase):
 
     def setUp(self):
-        self.link = links.EmbedID(3, 2)
+        self.link = links.EmbedID(3, 2, ignore_label=self.ignore_label)
+        self.link.ignore_label
         self.link.zerograds()
 
         self.W = self.link.W.data.copy()  # fixed on CPU
@@ -33,7 +36,10 @@ def check_forward(self, x_data):
 
         y_expect = numpy.empty_like(self.gy)
         for i in numpy.ndindex(self.x.shape):
-            y_expect[i] = self.W[int(self.x[i])]
+            if self.x[i] == -1:
+                y_expect[i] = 0
+            else:
+                y_expect[i] = self.W[int(self.x[i])]
 
         gradient_check.assert_allclose(y_expect, y.data, atol=0, rtol=0)