File size: 2,356 Bytes
2e4e201 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import Wav2Vec2Config
from .torch_utils import get_mask_from_lengths
from .wav2vec2 import Wav2Vec2Model
class Audio2MeshModel(nn.Module):
def __init__(
self,
config
):
super().__init__()
out_dim = config['out_dim']
latent_dim = config['latent_dim']
model_path = config['model_path']
only_last_fetures = config['only_last_fetures']
from_pretrained = config['from_pretrained']
self._only_last_features = only_last_fetures
self.audio_encoder_config = Wav2Vec2Config.from_pretrained(model_path, local_files_only=True)
if from_pretrained:
self.audio_encoder = Wav2Vec2Model.from_pretrained(model_path, local_files_only=True)
else:
self.audio_encoder = Wav2Vec2Model(self.audio_encoder_config)
self.audio_encoder.feature_extractor._freeze_parameters()
hidden_size = self.audio_encoder_config.hidden_size
self.in_fn = nn.Linear(hidden_size, latent_dim)
self.out_fn = nn.Linear(latent_dim, out_dim)
nn.init.constant_(self.out_fn.weight, 0)
nn.init.constant_(self.out_fn.bias, 0)
def forward(self, audio, label, audio_len=None):
attention_mask = ~get_mask_from_lengths(audio_len) if audio_len else None
seq_len = label.shape[1]
embeddings = self.audio_encoder(audio, seq_len=seq_len, output_hidden_states=True,
attention_mask=attention_mask)
if self._only_last_features:
hidden_states = embeddings.last_hidden_state
else:
hidden_states = sum(embeddings.hidden_states) / len(embeddings.hidden_states)
layer_in = self.in_fn(hidden_states)
out = self.out_fn(layer_in)
return out, None
def infer(self, input_value, seq_len):
embeddings = self.audio_encoder(input_value, seq_len=seq_len, output_hidden_states=True)
if self._only_last_features:
hidden_states = embeddings.last_hidden_state
else:
hidden_states = sum(embeddings.hidden_states) / len(embeddings.hidden_states)
layer_in = self.in_fn(hidden_states)
out = self.out_fn(layer_in)
return out
|