Repository: chainer/chainer

Issue:
Stateless LSTM link
I need a stateless-LSTM link like GRU. Stateful-LSTM link makes a code simple, but it is difficult to generate a sequence with beam search because we need to store top-k states to a priority queue.

We also need to design consistency of names.
- Stateful LSTM: `links.LSTM`
- Stateless LSTM: new!
- Stateful GRU: `links.StatefulGRU`
- Stateless GRU: `links.GRU`



Discussion:
Adding the beam-search code too would also be nice! I'm not aware of any libraries that have it available. 


Resolving patch:
diff --git a/chainer/links/__init__.py b/chainer/links/__init__.py
--- a/chainer/links/__init__.py
+++ b/chainer/links/__init__.py
@@ -32,6 +32,7 @@
 InceptionBN = inceptionbn.InceptionBN
 Linear = linear.Linear
 LSTM = lstm.LSTM
+StatelessLSTM = lstm.StatelessLSTM
 MLPConvolution2D = mlp_convolution_2d.MLPConvolution2D
 Parameter = parameter.Parameter
 
diff --git a/chainer/links/connection/lstm.py b/chainer/links/connection/lstm.py
--- a/chainer/links/connection/lstm.py
+++ b/chainer/links/connection/lstm.py
@@ -7,7 +7,77 @@
 from chainer import variable
 
 
-class LSTM(link.Chain):
+class LSTMBase(link.Chain):
+
+    def __init__(self, in_size, out_size,
+                 lateral_init=None, upward_init=None,
+                 bias_init=0, forget_bias_init=0):
+        super(LSTMBase, self).__init__(
+            upward=linear.Linear(in_size, 4 * out_size, initialW=0),
+            lateral=linear.Linear(out_size, 4 * out_size,
+                                  initialW=0, nobias=True),
+        )
+        self.state_size = out_size
+
+        for i in six.moves.range(0, 4 * out_size, out_size):
+            initializers.init_weight(
+                self.lateral.W.data[i:i + out_size, :], lateral_init)
+            initializers.init_weight(
+                self.upward.W.data[i:i + out_size, :], upward_init)
+
+        a, i, f, o = lstm._extract_gates(
+            self.upward.b.data.reshape(1, 4 * out_size, 1))
+        initializers.init_weight(a, bias_init)
+        initializers.init_weight(i, bias_init)
+        initializers.init_weight(f, forget_bias_init)
+        initializers.init_weight(o, bias_init)
+
+
+class StatelessLSTM(LSTMBase):
+
+    """Stateless LSTM layer.
+
+    This is a fully-connected LSTM layer as a chain. Unlike the
+    :func:`~chainer.functions.lstm` function, this chain holds upward and
+    lateral connections as child links. This link doesn't keep cell and
+    hidden states.
+
+    Args:
+        in_size (int): Dimensionality of input vectors.
+        out_size (int): Dimensionality of output vectors.
+
+    Attributes:
+        upward (chainer.links.Linear): Linear layer of upward connections.
+        lateral (chainer.links.Linear): Linear layer of lateral connections.
+
+    """
+
+    def __call__(self, c, h, x):
+        """Returns new cell state and updated output of LSTM.
+
+        Args:
+            c (~chainer.Variable): Cell states of LSTM units.
+            h (~chainer.Variable): Output at the previous timestep.
+            x (~chainer.Variable): A new batch from the input sequence.
+
+        Returns:
+            tuple of ~chainer.Variable: Returns ``(c_new, h_new)``, where
+                ``c_new`` represents new cell state, and ``h_new`` is updated
+                output of LSTM units.
+
+        """
+        lstm_in = self.upward(x)
+        if h is not None:
+            lstm_in += self.lateral(h)
+        if c is None:
+            xp = self.xp
+            c = variable.Variable(
+                xp.zeros((len(x.data), self.state_size), dtype=x.data.dtype),
+                volatile='auto')
+        return lstm.lstm(c, lstm_in)
+
+
+class LSTM(LSTMBase):
 
     """Fully-connected LSTM layer.
 
@@ -54,30 +124,10 @@ class LSTM(link.Chain):
 
     """
 
-    def __init__(self, in_size, out_size,
-                 lateral_init=None, upward_init=None,
-                 bias_init=0, forget_bias_init=0):
-        super(LSTM, self).__init__(
-            upward=linear.Linear(in_size, 4 * out_size, initialW=0),
-            lateral=linear.Linear(out_size, 4 * out_size,
-                                  initialW=0, nobias=True),
-        )
-        self.state_size = out_size
+    def __init__(self, in_size, out_size, **kwargs):
+        super(LSTM, self).__init__(in_size, out_size, **kwargs)
         self.reset_state()
 
-        for i in six.moves.range(0, 4 * out_size, out_size):
-            initializers.init_weight(
-                self.lateral.W.data[i:i + out_size, :], lateral_init)
-            initializers.init_weight(
-                self.upward.W.data[i:i + out_size, :], upward_init)
-
-        a, i, f, o = lstm._extract_gates(
-            self.upward.b.data.reshape(1, 4 * out_size, 1))
-        initializers.init_weight(a, bias_init)
-        initializers.init_weight(i, bias_init)
-        initializers.init_weight(f, forget_bias_init)
-        initializers.init_weight(o, bias_init)
-
     def to_cpu(self):
         super(LSTM, self).to_cpu()
         if self.c is not None:


Test patch:
diff --git a/tests/chainer_tests/links_tests/connection_tests/test_lstm.py b/tests/chainer_tests/links_tests/connection_tests/test_lstm.py
--- a/tests/chainer_tests/links_tests/connection_tests/test_lstm.py
+++ b/tests/chainer_tests/links_tests/connection_tests/test_lstm.py
@@ -147,4 +147,50 @@ def test_to_cpu_to_gpu_gpu(self):
         self.check_to_cpu_to_gpu(self.link.h)
 
 
+@testing.parameterize(
+    {'in_size': 10, 'out_size': 10},
+    {'in_size': 10, 'out_size': 40},
+)
+class TestStatelessLSTM(unittest.TestCase):
+
+    def setUp(self):
+        self.link = links.StatelessLSTM(self.in_size, self.out_size)
+        upward = self.link.upward.W.data
+        upward[...] = numpy.random.uniform(-1, 1, upward.shape)
+        lateral = self.link.lateral.W.data
+        lateral[...] = numpy.random.uniform(-1, 1, lateral.shape)
+        self.link.zerograds()
+
+        self.upward = upward.copy()  # fixed on CPU
+        self.lateral = lateral.copy()  # fixed on CPU
+
+        x_shape = (4, self.in_size)
+        self.x = numpy.random.uniform(-1, 1, x_shape).astype(numpy.float32)
+
+    def check_forward(self, x_data):
+        xp = self.link.xp
+        x = chainer.Variable(x_data)
+        c1, h1 = self.link(None, None, x)
+        c0 = chainer.Variable(xp.zeros((len(self.x), self.out_size),
+                                       dtype=self.x.dtype))
+        c1_expect, h1_expect = functions.lstm(c0, self.link.upward(x))
+        gradient_check.assert_allclose(h1.data, h1_expect.data)
+        gradient_check.assert_allclose(c1.data, c1_expect.data)
+
+        c2, h2 = self.link(c1, h1, x)
+        c2_expect, h2_expect = \
+            functions.lstm(c1_expect,
+                           self.link.upward(x) + self.link.lateral(h1))
+        gradient_check.assert_allclose(h2.data, h2_expect.data)
+        gradient_check.assert_allclose(c2.data, c2_expect.data)
+
+    def test_forward_cpu(self):
+        self.check_forward(self.x)
+
+    @attr.gpu
+    def test_forward_gpu(self):
+        self.link.to_gpu()
+        self.check_forward(cuda.to_gpu(self.x))
+
+
 testing.run_module(__name__, __file__)