File size: 1,948 Bytes
ad16788 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
# encoding: utf-8
"""Class Declaration of Transformer's Positionwise Feedforward."""
import chainer
import chainer.functions as F
import chainer.links as L
import numpy as np
class PositionwiseFeedForward(chainer.Chain):
"""Positionwise feed forward.
Args:
:param int idim: input dimenstion
:param int hidden_units: number of hidden units
:param float dropout_rate: dropout rate
"""
def __init__(
self, n_units, d_units=0, dropout=0.1, initialW=None, initial_bias=None
):
"""Initialize PositionwiseFeedForward.
Args:
n_units (int): Input dimension.
d_units (int, optional): Output dimension of hidden layer.
dropout (float, optional): Dropout ratio.
initialW (int, optional): Initializer to initialize the weight.
initial_bias (bool, optional): Initializer to initialize the bias.
"""
super(PositionwiseFeedForward, self).__init__()
n_inner_units = d_units if d_units > 0 else n_units * 4
with self.init_scope():
stvd = 1.0 / np.sqrt(n_units)
self.w_1 = L.Linear(
n_units,
n_inner_units,
initialW=initialW(scale=stvd),
initial_bias=initial_bias(scale=stvd),
)
stvd = 1.0 / np.sqrt(n_inner_units)
self.w_2 = L.Linear(
n_inner_units,
n_units,
initialW=initialW(scale=stvd),
initial_bias=initial_bias(scale=stvd),
)
self.act = F.relu
self.dropout = dropout
def __call__(self, e):
"""Initialize PositionwiseFeedForward.
Args:
e (chainer.Variable): Input variable.
Return:
chainer.Variable: Output variable.
"""
e = F.dropout(self.act(self.w_1(e)), self.dropout)
return self.w_2(e)
|