Repository: chainer/chainer

Issue:
Implement Gated Recurrent Unit
Suggested by @chrisemoody at twitter. It is desirable to implement GRU unit to cover various kind of architecture.



Discussion:
+1

+1 :)


Resolving patch:
diff --git a/chainer/links/__init__.py b/chainer/links/__init__.py
--- a/chainer/links/__init__.py
+++ b/chainer/links/__init__.py
@@ -4,6 +4,7 @@
 from chainer.links.connection import bilinear
 from chainer.links.connection import convolution_2d
 from chainer.links.connection import embed_id
+from chainer.links.connection import gru
 from chainer.links.connection import inception
 from chainer.links.connection import inceptionbn
 from chainer.links.connection import linear
@@ -21,6 +22,7 @@
 Bilinear = bilinear.Bilinear
 Convolution2D = convolution_2d.Convolution2D
 EmbedID = embed_id.EmbedID
+GRU = gru.GRU
 Inception = inception.Inception
 InceptionBN = inceptionbn.InceptionBN
 Linear = linear.Linear
diff --git a/chainer/links/connection/gru.py b/chainer/links/connection/gru.py
new file mode 100644
--- /dev/null
+++ b/chainer/links/connection/gru.py
@@ -0,0 +1,58 @@
+from chainer.functions.activation import sigmoid
+from chainer.functions.activation import tanh
+from chainer import link
+from chainer.links.connection import linear
+
+
+class GRU(link.Chain):
+
+    """Gated Recurrent Unit function (GRU).
+
+    GRU function has six parameters :math:`W_r`, :math:`W_z`, :math:`W`,
+    :math:`U_r`, :math:`U_z`, and :math:`U`. All these parameters are
+    :math:`n \\times n` matricies, where :math:`n` is the dimension of
+    hidden vectors.
+
+    Given two inputs a previous hidden vector :math:`h` and an input vector
+    :math:`x`, GRU returns the next hidden vector :math:`h'` defined as
+
+    .. math::
+
+       r &=& \\sigma(W_r x + U_r h), \\\\
+       z &=& \\sigma(W_z x + U_z h), \\\\
+       \\bar{h} &=& \\tanh(W x + U (r \\odot h)), \\\\
+       h' &=& (1 - z) \\odot h + z \\odot \\bar{h},
+
+    where :math:`\\sigma` is the sigmoid function, and :math:`\\odot` is the
+    element-wise product.
+
+    Args:
+        n_units(int): Dimension of input vector :math:`x`, and hidden vector
+            :math:`h`.
+
+    See:
+        - `On the Properties of Neural Machine Translation: Encoder-Decoder
+          Approaches <http://www.aclweb.org/anthology/W14-4012>`_
+          [Cho+, SSST2014].
+        - `Empirical Evaluation of Gated Recurrent Neural Networks on Sequence
+          Modeling <http://arxiv.org/abs/1412.3555>`_
+          [Chung+NIPS2014 DLWorkshop].
+
+    """
+
+    def __init__(self, n_units):
+        super(GRU, self).__init__(
+            W_r=linear.Linear(n_units, n_units),
+            U_r=linear.Linear(n_units, n_units),
+            W_z=linear.Linear(n_units, n_units),
+            U_z=linear.Linear(n_units, n_units),
+            W=linear.Linear(n_units, n_units),
+            U=linear.Linear(n_units, n_units),
+        )
+
+    def __call__(self, h, x):
+        r = sigmoid.sigmoid(self.W_r(x) + self.U_r(h))
+        z = sigmoid.sigmoid(self.W_z(x) + self.U_z(h))
+        h_bar = tanh.tanh(self.W(x) + self.U(r * h))
+        h_new = (1 - z) * h + z * h_bar
+        return h_new


Test patch:
diff --git a/tests/chainer_tests/links_tests/connection_tests/test_gru.py b/tests/chainer_tests/links_tests/connection_tests/test_gru.py
new file mode 100644
--- /dev/null
+++ b/tests/chainer_tests/links_tests/connection_tests/test_gru.py
@@ -0,0 +1,79 @@
+import unittest
+
+import numpy
+
+import chainer
+from chainer import cuda
+from chainer import gradient_check
+from chainer import links
+from chainer import testing
+from chainer.testing import attr
+
+
+def _sigmoid(x):
+    xp = cuda.get_array_module(x)
+    return 1 / (1 + xp.exp(-x))
+
+
+def _gru(func, h, x):
+    xp = cuda.get_array_module(h, x)
+
+    r = _sigmoid(x.dot(func.W_r.W.data.T) + h.dot(func.U_r.W.data.T))
+    z = _sigmoid(x.dot(func.W_z.W.data.T) + h.dot(func.U_z.W.data.T))
+    h_bar = xp.tanh(x.dot(func.W.W.data.T) + (r * h).dot(func.U.W.data.T))
+    y = (1 - z) * h + z * h_bar
+    return y
+
+
+class TestGRU(unittest.TestCase):
+
+    def setUp(self):
+        self.link = links.GRU(8)
+        self.x = numpy.random.uniform(-1, 1, (3, 8)).astype(numpy.float32)
+        self.h = numpy.random.uniform(-1, 1, (3, 8)).astype(numpy.float32)
+        self.gy = numpy.random.uniform(-1, 1, (3, 8)).astype(numpy.float32)
+
+    def check_forward(self, h_data, x_data):
+        h = chainer.Variable(h_data)
+        x = chainer.Variable(x_data)
+        y = self.link(h, x)
+        self.assertEqual(y.data.dtype, numpy.float32)
+
+        y_expect = _gru(self.link, h_data, x_data)
+        gradient_check.assert_allclose(y_expect, y.data)
+
+    def test_forward_cpu(self):
+        self.check_forward(self.h, self.x)
+
+    @attr.gpu
+    def test_forward_gpu(self):
+        self.link.to_gpu()
+        self.check_forward(cuda.to_gpu(self.h),
+                           cuda.to_gpu(self.x))
+
+    def check_backward(self, h_data, x_data, y_grad):
+        h = chainer.Variable(h_data)
+        x = chainer.Variable(x_data)
+        y = self.link(h, x)
+
+        y.grad = y_grad
+        y.backward()
+
+        f = lambda: (_gru(self.link, h.data, x.data),)
+        gh, gx = gradient_check.numerical_grad(f, (h.data, x.data), (y.grad,))
+
+        gradient_check.assert_allclose(gh, h.grad, atol=1e-3)
+        gradient_check.assert_allclose(gx, x.grad, atol=1e-3)
+
+    def test_backward_cpu(self):
+        self.check_backward(self.h, self.x, self.gy)
+
+    @attr.gpu
+    def test_backward_gpu(self):
+        self.link.to_gpu()
+        self.check_backward(cuda.to_gpu(self.h),
+                            cuda.to_gpu(self.x),
+                            cuda.to_gpu(self.gy))
+
+
+testing.run_module(__name__, __file__)