Spaces:
Build error
Build error
# cp from https://github.com/lifeiteng/vall-e/blob/main/valle/modules/embedding.py | |
# Copyright 2023 (authors: Feiteng Li) | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
import math | |
import torch | |
import torch.nn as nn | |
class TokenEmbedding(nn.Module): | |
def __init__( | |
self, | |
dim_model: int, | |
vocab_size: int, | |
dropout: float = 0.0, | |
): | |
super().__init__() | |
self.vocab_size = vocab_size | |
self.dim_model = dim_model | |
self.dropout = torch.nn.Dropout(p=dropout) | |
self.word_embeddings = nn.Embedding(self.vocab_size, self.dim_model) | |
def weight(self) -> torch.Tensor: | |
return self.word_embeddings.weight | |
def embedding(self, index: int) -> torch.Tensor: | |
return self.word_embeddings.weight[index : index + 1] | |
def forward(self, x: torch.Tensor): | |
X = self.word_embeddings(x) | |
X = self.dropout(X) | |
return X | |
class SinePositionalEmbedding(nn.Module): | |
def __init__( | |
self, | |
dim_model: int, | |
dropout: float = 0.0, | |
scale: bool = False, | |
alpha: bool = False, | |
): | |
super().__init__() | |
self.dim_model = dim_model | |
self.x_scale = math.sqrt(dim_model) if scale else 1.0 | |
self.alpha = nn.Parameter(torch.ones(1), requires_grad=alpha) | |
self.dropout = torch.nn.Dropout(p=dropout) | |
self.reverse = False | |
self.pe = None | |
self.extend_pe(torch.tensor(0.0).expand(1, 4000)) | |
def extend_pe(self, x): | |
"""Reset the positional encodings.""" | |
if self.pe is not None: | |
if self.pe.size(1) >= x.size(1): | |
if self.pe.dtype != x.dtype or self.pe.device != x.device: | |
self.pe = self.pe.to(dtype=x.dtype, device=x.device) | |
return | |
pe = torch.zeros(x.size(1), self.dim_model) | |
if self.reverse: | |
position = torch.arange( | |
x.size(1) - 1, -1, -1.0, dtype=torch.float32 | |
).unsqueeze(1) | |
else: | |
position = torch.arange( | |
0, x.size(1), dtype=torch.float32 | |
).unsqueeze(1) | |
div_term = torch.exp( | |
torch.arange(0, self.dim_model, 2, dtype=torch.float32) | |
* -(math.log(10000.0) / self.dim_model) | |
) | |
pe[:, 0::2] = torch.sin(position * div_term) | |
pe[:, 1::2] = torch.cos(position * div_term) | |
pe = pe.unsqueeze(0) | |
self.pe = pe.to(device=x.device, dtype=x.dtype).detach() | |
def forward(self, x: torch.Tensor) -> torch.Tensor: | |
self.extend_pe(x) | |
output = x.unsqueeze(-1) if x.ndim == 2 else x | |
output = output * self.x_scale + self.alpha * self.pe[:, : x.size(1)] | |
return self.dropout(output) |