Repository: chainer/chainer

Issue:
L.LSTM is not initialized correctly
According to the documentation (https://docs.chainer.org/en/latest/reference/generated/chainer.links.LSTM.html):

> **bias_init** – A callable that takes numpy.ndarray or cupy.ndarray and edits its value It is used for initialization of the biases of cell input, input gate and output gate.and gates of the upward connection. May be a scalar, in that case, the bias is initialized by this value. If it is None, the cell-input bias is initialized to zero.
> **forget_bias_init** – A callable that takes numpy.ndarray or cupy.ndarray and edits its value It is used for initialization of the biases of the forget gate of the upward connection. May be a scalar, in that case, the bias is initialized by this value. If it is None, the forget bias is initialized to one.

**Code**:
```
import numpy
import chainer.links as L

l = L.LSTM(2, lateral_init=0, upward_init=0, bias_init=None, forget_bias_init=None)
#l = L.LSTM(2, lateral_init=0, upward_init=0, bias_init=2, forget_bias_init=3)
y = l(numpy.random.random((2, 2)).astype('f'))
for n, p in sorted(l.namedparams()):
    print(n)
    print(p)
```

**Results** (on current master e0ce78850de29014df6f7db86b68ac45fc57541d):
* `l = L.LSTM(2, lateral_init=0, upward_init=0, bias_init=None, forget_bias_init=None)`
```
/lateral/W
variable W([[ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]])
/upward/W
variable W([[ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]])
/upward/b
variable b([ 0.90749902 -0.21517469 -0.0032583   0.30968115  0.22123729
            -0.22819807 -1.55126989 -0.85002041])

```

* `l = L.LSTM(2, lateral_init=0, upward_init=0, bias_init=2, forget_bias_init=3)`
```
/lateral/W
variable W([[ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]])
/upward/W
variable W([[ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]
            [ 0.  0.]])
/upward/b
variable b([ 2.  2.  3.  2.  2.  2.  3.  2.])

```


Discussion:

Resolving patch:
diff --git a/chainer/links/connection/lstm.py b/chainer/links/connection/lstm.py
--- a/chainer/links/connection/lstm.py
+++ b/chainer/links/connection/lstm.py
@@ -11,18 +11,18 @@
 from chainer import variable
 
 
-def _init_weight(weights, initializer):
-    initializers._get_initializer(initializer)(weights)
-
-
 class LSTMBase(link.Chain):
 
     def __init__(self, in_size, out_size=None, lateral_init=None,
-                 upward_init=None, bias_init=0, forget_bias_init=1):
+                 upward_init=None, bias_init=None, forget_bias_init=None):
         if out_size is None:
             out_size, in_size = in_size, None
 
         super(LSTMBase, self).__init__()
+        if bias_init is None:
+            bias_init = 0
+        if forget_bias_init is None:
+            forget_bias_init = 1
         self.state_size = out_size
         self.lateral_init = lateral_init
         self.upward_init = upward_init
@@ -39,6 +39,8 @@ def __init__(self, in_size, out_size=None, lateral_init=None,
     def _initialize_params(self):
         lateral_init = initializers._get_initializer(self.lateral_init)
         upward_init = initializers._get_initializer(self.upward_init)
+        bias_init = initializers._get_initializer(self.bias_init)
+        forget_bias_init = initializers._get_initializer(self.forget_bias_init)
 
         for i in six.moves.range(0, 4 * self.state_size, self.state_size):
             lateral_init(self.lateral.W.data[i:i + self.state_size, :])
@@ -46,10 +48,11 @@ def _initialize_params(self):
 
         a, i, f, o = lstm._extract_gates(
             self.upward.b.data.reshape(1, 4 * self.state_size, 1))
-        _init_weight(a, self.bias_init)
-        _init_weight(i, self.bias_init)
-        _init_weight(f, self.forget_bias_init)
-        _init_weight(o, self.bias_init)
+
+        bias_init(a)
+        bias_init(i)
+        forget_bias_init(f)
+        bias_init(o)
 
 
 class StatelessLSTM(LSTMBase):
@@ -229,10 +232,13 @@ class LSTM(LSTMBase):
             (1, 20)
     """
 
-    def __init__(self, in_size, out_size=None, **kwargs):
+    def __init__(self, in_size, out_size=None, lateral_init=None,
+                 upward_init=None, bias_init=None, forget_bias_init=None):
         if out_size is None:
             in_size, out_size = None, in_size
-        super(LSTM, self).__init__(in_size, out_size, **kwargs)
+        super(LSTM, self).__init__(
+            in_size, out_size, lateral_init, upward_init, bias_init,
+            forget_bias_init)
         self.reset_state()
 
     def to_cpu(self):


Test patch:
diff --git a/tests/chainer_tests/links_tests/connection_tests/test_lstm.py b/tests/chainer_tests/links_tests/connection_tests/test_lstm.py
--- a/tests/chainer_tests/links_tests/connection_tests/test_lstm.py
+++ b/tests/chainer_tests/links_tests/connection_tests/test_lstm.py
@@ -239,6 +239,15 @@ def test_forward_invalid_size_gpu(self):
                                         cuda.to_gpu(self.x2))
 
 
+class TestLSTMInitialize(unittest.TestCase):
+
+    def test_initialize_bias_default(self):
+        link = links.LSTM(2, 3)
+        numpy.testing.assert_array_equal(
+            link.upward.b.data,
+            numpy.array([0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0]))
+
+
 @testing.parameterize(*testing.product_dict(
     [
         {'in_size': 10, 'out_size': 10},