Spaces:
Build error
Build error
| # Written by Shigeki Karita, 2019 | |
| # Published under Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) | |
| # Adapted by Florian Lux, 2021 | |
| """Multi-Head Attention layer definition.""" | |
| import math | |
| import numpy | |
| import torch | |
| from torch import nn | |
| from Utility.utils import make_non_pad_mask | |
| class MultiHeadedAttention(nn.Module): | |
| """ | |
| Multi-Head Attention layer. | |
| Args: | |
| n_head (int): The number of heads. | |
| n_feat (int): The number of features. | |
| dropout_rate (float): Dropout rate. | |
| """ | |
| def __init__(self, n_head, n_feat, dropout_rate): | |
| """ | |
| Construct an MultiHeadedAttention object. | |
| """ | |
| super(MultiHeadedAttention, self).__init__() | |
| assert n_feat % n_head == 0 | |
| # We assume d_v always equals d_k | |
| self.d_k = n_feat // n_head | |
| self.h = n_head | |
| self.linear_q = nn.Linear(n_feat, n_feat) | |
| self.linear_k = nn.Linear(n_feat, n_feat) | |
| self.linear_v = nn.Linear(n_feat, n_feat) | |
| self.linear_out = nn.Linear(n_feat, n_feat) | |
| self.attn = None | |
| self.dropout = nn.Dropout(p=dropout_rate) | |
| def forward_qkv(self, query, key, value): | |
| """ | |
| Transform query, key and value. | |
| Args: | |
| query (torch.Tensor): Query tensor (#batch, time1, size). | |
| key (torch.Tensor): Key tensor (#batch, time2, size). | |
| value (torch.Tensor): Value tensor (#batch, time2, size). | |
| Returns: | |
| torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). | |
| torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). | |
| torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). | |
| """ | |
| n_batch = query.size(0) | |
| q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) | |
| k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) | |
| v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) | |
| q = q.transpose(1, 2) # (batch, head, time1, d_k) | |
| k = k.transpose(1, 2) # (batch, head, time2, d_k) | |
| v = v.transpose(1, 2) # (batch, head, time2, d_k) | |
| return q, k, v | |
| def forward_attention(self, value, scores, mask): | |
| """ | |
| Compute attention context vector. | |
| Args: | |
| value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). | |
| scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). | |
| mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). | |
| Returns: | |
| torch.Tensor: Transformed value (#batch, time1, d_model) | |
| weighted by the attention score (#batch, time1, time2). | |
| """ | |
| n_batch = value.size(0) | |
| if mask is not None: | |
| mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) | |
| min_value = float(numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min) | |
| scores = scores.masked_fill(mask, min_value) | |
| self.attn = torch.softmax(scores, dim=-1).masked_fill(mask, 0.0) # (batch, head, time1, time2) | |
| else: | |
| self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) | |
| p_attn = self.dropout(self.attn) | |
| x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) | |
| x = (x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k)) # (batch, time1, d_model) | |
| return self.linear_out(x) # (batch, time1, d_model) | |
| def forward(self, query, key, value, mask): | |
| """ | |
| Compute scaled dot product attention. | |
| Args: | |
| query (torch.Tensor): Query tensor (#batch, time1, size). | |
| key (torch.Tensor): Key tensor (#batch, time2, size). | |
| value (torch.Tensor): Value tensor (#batch, time2, size). | |
| mask (torch.Tensor): Mask tensor (#batch, 1, time2) or | |
| (#batch, time1, time2). | |
| Returns: | |
| torch.Tensor: Output tensor (#batch, time1, d_model). | |
| """ | |
| q, k, v = self.forward_qkv(query, key, value) | |
| scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) | |
| return self.forward_attention(v, scores, mask) | |
| class RelPositionMultiHeadedAttention(MultiHeadedAttention): | |
| """ | |
| Multi-Head Attention layer with relative position encoding. | |
| Details can be found in https://github.com/espnet/espnet/pull/2816. | |
| Paper: https://arxiv.org/abs/1901.02860 | |
| Args: | |
| n_head (int): The number of heads. | |
| n_feat (int): The number of features. | |
| dropout_rate (float): Dropout rate. | |
| zero_triu (bool): Whether to zero the upper triangular part of attention matrix. | |
| """ | |
| def __init__(self, n_head, n_feat, dropout_rate, zero_triu=False): | |
| """Construct an RelPositionMultiHeadedAttention object.""" | |
| super().__init__(n_head, n_feat, dropout_rate) | |
| self.zero_triu = zero_triu | |
| # linear transformation for positional encoding | |
| self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) | |
| # these two learnable bias are used in matrix c and matrix d | |
| # as described in https://arxiv.org/abs/1901.02860 Section 3.3 | |
| self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) | |
| self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) | |
| torch.nn.init.xavier_uniform_(self.pos_bias_u) | |
| torch.nn.init.xavier_uniform_(self.pos_bias_v) | |
| def rel_shift(self, x): | |
| """ | |
| Compute relative positional encoding. | |
| Args: | |
| x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1). | |
| time1 means the length of query vector. | |
| Returns: | |
| torch.Tensor: Output tensor. | |
| """ | |
| zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) | |
| x_padded = torch.cat([zero_pad, x], dim=-1) | |
| x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) | |
| x = x_padded[:, :, 1:].view_as(x)[:, :, :, : x.size(-1) // 2 + 1] # only keep the positions from 0 to time2 | |
| if self.zero_triu: | |
| ones = torch.ones((x.size(2), x.size(3)), device=x.device) | |
| x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] | |
| return x | |
| def forward(self, query, key, value, pos_emb, mask): | |
| """ | |
| Compute 'Scaled Dot Product Attention' with rel. positional encoding. | |
| Args: | |
| query (torch.Tensor): Query tensor (#batch, time1, size). | |
| key (torch.Tensor): Key tensor (#batch, time2, size). | |
| value (torch.Tensor): Value tensor (#batch, time2, size). | |
| pos_emb (torch.Tensor): Positional embedding tensor | |
| (#batch, 2*time1-1, size). | |
| mask (torch.Tensor): Mask tensor (#batch, 1, time2) or | |
| (#batch, time1, time2). | |
| Returns: | |
| torch.Tensor: Output tensor (#batch, time1, d_model). | |
| """ | |
| q, k, v = self.forward_qkv(query, key, value) | |
| q = q.transpose(1, 2) # (batch, time1, head, d_k) | |
| n_batch_pos = pos_emb.size(0) | |
| p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) | |
| p = p.transpose(1, 2) # (batch, head, 2*time1-1, d_k) | |
| # (batch, head, time1, d_k) | |
| q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) | |
| # (batch, head, time1, d_k) | |
| q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) | |
| # compute attention score | |
| # first compute matrix a and matrix c | |
| # as described in https://arxiv.org/abs/1901.02860 Section 3.3 | |
| # (batch, head, time1, time2) | |
| matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) | |
| # compute matrix b and matrix d | |
| # (batch, head, time1, 2*time1-1) | |
| matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) | |
| matrix_bd = self.rel_shift(matrix_bd) | |
| scores = (matrix_ac + matrix_bd) / math.sqrt(self.d_k) # (batch, head, time1, time2) | |
| return self.forward_attention(v, scores, mask) | |
| class GuidedAttentionLoss(torch.nn.Module): | |
| """ | |
| Guided attention loss function module. | |
| This module calculates the guided attention loss described | |
| in `Efficiently Trainable Text-to-Speech System Based | |
| on Deep Convolutional Networks with Guided Attention`_, | |
| which forces the attention to be diagonal. | |
| .. _`Efficiently Trainable Text-to-Speech System | |
| Based on Deep Convolutional Networks with Guided Attention`: | |
| https://arxiv.org/abs/1710.08969 | |
| """ | |
| def __init__(self, sigma=0.4, alpha=1.0): | |
| """ | |
| Initialize guided attention loss module. | |
| Args: | |
| sigma (float, optional): Standard deviation to control | |
| how close attention to a diagonal. | |
| alpha (float, optional): Scaling coefficient (lambda). | |
| reset_always (bool, optional): Whether to always reset masks. | |
| """ | |
| super(GuidedAttentionLoss, self).__init__() | |
| self.sigma = sigma | |
| self.alpha = alpha | |
| self.guided_attn_masks = None | |
| self.masks = None | |
| def _reset_masks(self): | |
| self.guided_attn_masks = None | |
| self.masks = None | |
| def forward(self, att_ws, ilens, olens): | |
| """ | |
| Calculate forward propagation. | |
| Args: | |
| att_ws (Tensor): Batch of attention weights (B, T_max_out, T_max_in). | |
| ilens (LongTensor): Batch of input lenghts (B,). | |
| olens (LongTensor): Batch of output lenghts (B,). | |
| Returns: | |
| Tensor: Guided attention loss value. | |
| """ | |
| self._reset_masks() | |
| self.guided_attn_masks = self._make_guided_attention_masks(ilens, olens).to(att_ws.device) | |
| self.masks = self._make_masks(ilens, olens).to(att_ws.device) | |
| losses = self.guided_attn_masks * att_ws | |
| loss = torch.mean(losses.masked_select(self.masks)) | |
| self._reset_masks() | |
| return self.alpha * loss | |
| def _make_guided_attention_masks(self, ilens, olens): | |
| n_batches = len(ilens) | |
| max_ilen = max(ilens) | |
| max_olen = max(olens) | |
| guided_attn_masks = torch.zeros((n_batches, max_olen, max_ilen), device=ilens.device) | |
| for idx, (ilen, olen) in enumerate(zip(ilens, olens)): | |
| guided_attn_masks[idx, :olen, :ilen] = self._make_guided_attention_mask(ilen, olen, self.sigma) | |
| return guided_attn_masks | |
| def _make_guided_attention_mask(ilen, olen, sigma): | |
| """ | |
| Make guided attention mask. | |
| """ | |
| grid_x, grid_y = torch.meshgrid(torch.arange(olen, device=olen.device).float(), torch.arange(ilen, device=ilen.device).float()) | |
| return 1.0 - torch.exp(-((grid_y / ilen - grid_x / olen) ** 2) / (2 * (sigma ** 2))) | |
| def _make_masks(ilens, olens): | |
| """ | |
| Make masks indicating non-padded part. | |
| Args: | |
| ilens (LongTensor or List): Batch of lengths (B,). | |
| olens (LongTensor or List): Batch of lengths (B,). | |
| Returns: | |
| Tensor: Mask tensor indicating non-padded part. | |
| dtype=torch.uint8 in PyTorch 1.2- | |
| dtype=torch.bool in PyTorch 1.2+ (including 1.2) | |
| """ | |
| in_masks = make_non_pad_mask(ilens, device=ilens.device) # (B, T_in) | |
| out_masks = make_non_pad_mask(olens, device=olens.device) # (B, T_out) | |
| return out_masks.unsqueeze(-1) & in_masks.unsqueeze(-2) # (B, T_out, T_in) | |
| class GuidedMultiHeadAttentionLoss(GuidedAttentionLoss): | |
| """ | |
| Guided attention loss function module for multi head attention. | |
| Args: | |
| sigma (float, optional): Standard deviation to control | |
| how close attention to a diagonal. | |
| alpha (float, optional): Scaling coefficient (lambda). | |
| reset_always (bool, optional): Whether to always reset masks. | |
| """ | |
| def forward(self, att_ws, ilens, olens): | |
| """ | |
| Calculate forward propagation. | |
| Args: | |
| att_ws (Tensor): | |
| Batch of multi head attention weights (B, H, T_max_out, T_max_in). | |
| ilens (LongTensor): Batch of input lenghts (B,). | |
| olens (LongTensor): Batch of output lenghts (B,). | |
| Returns: | |
| Tensor: Guided attention loss value. | |
| """ | |
| if self.guided_attn_masks is None: | |
| self.guided_attn_masks = (self._make_guided_attention_masks(ilens, olens).to(att_ws.device).unsqueeze(1)) | |
| if self.masks is None: | |
| self.masks = self._make_masks(ilens, olens).to(att_ws.device).unsqueeze(1) | |
| losses = self.guided_attn_masks * att_ws | |
| loss = torch.mean(losses.masked_select(self.masks)) | |
| if self.reset_always: | |
| self._reset_masks() | |
| return self.alpha * loss | |