fairseq/fairseq/modules/lstm_cell_with_zoneout.py

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import torch.nn as nn


class LSTMCellWithZoneOut(nn.Module):
    """
    Zoneout: Regularizing RNNs by Randomly Preserving Hidden Activations
    https://arxiv.org/abs/1606.01305
    """

    def __init__(self, prob: float, input_size: int, hidden_size: int,
                 bias: bool = True):
        super(LSTMCellWithZoneOut, self).__init__()
        self.lstm_cell = nn.LSTMCell(input_size, hidden_size, bias=bias)
        self.prob = prob
        if prob > 1.0 or prob < 0.0:
            raise ValueError("zoneout probability must be in the range from "
                             "0.0 to 1.0.")

    def zoneout(self, h, next_h, prob):
        if isinstance(h, tuple):
            return tuple(
                [self.zoneout(h[i], next_h[i], prob) for i in range(len(h))]
            )

        if self.training:
            mask = h.new_zeros(*h.size()).bernoulli_(prob)
            return mask * h + (1 - mask) * next_h

        return prob * h + (1 - prob) * next_h

    def forward(self, x, h):
        return self.zoneout(h, self.lstm_cell(x, h), self.prob)