|
import os |
|
from glob import glob |
|
|
|
import torch |
|
import torchaudio |
|
import numpy as np |
|
from scipy.io.wavfile import read |
|
|
|
from utils.stft import STFT |
|
|
|
|
|
def load_wav_to_torch(full_path): |
|
sampling_rate, data = read(full_path) |
|
if data.dtype == np.int32: |
|
norm_fix = 2 ** 31 |
|
elif data.dtype == np.int16: |
|
norm_fix = 2 ** 15 |
|
elif data.dtype == np.float16 or data.dtype == np.float32: |
|
norm_fix = 1. |
|
else: |
|
raise NotImplemented(f"Provided data dtype not supported: {data.dtype}") |
|
return (torch.FloatTensor(data.astype(np.float32)) / norm_fix, sampling_rate) |
|
|
|
|
|
def load_audio(audiopath, sampling_rate): |
|
if audiopath[-4:] == '.wav': |
|
audio, lsr = load_wav_to_torch(audiopath) |
|
elif audiopath[-4:] == '.mp3': |
|
|
|
from pyfastmp3decoder.mp3decoder import load_mp3 |
|
audio, lsr = load_mp3(audiopath, sampling_rate) |
|
audio = torch.FloatTensor(audio) |
|
|
|
|
|
if len(audio.shape) > 1: |
|
if audio.shape[0] < 5: |
|
audio = audio[0] |
|
else: |
|
assert audio.shape[1] < 5 |
|
audio = audio[:, 0] |
|
|
|
if lsr != sampling_rate: |
|
audio = torchaudio.functional.resample(audio, lsr, sampling_rate) |
|
|
|
|
|
|
|
if torch.any(audio > 2) or not torch.any(audio < 0): |
|
print(f"Error with {audiopath}. Max={audio.max()} min={audio.min()}") |
|
audio.clip_(-1, 1) |
|
|
|
return audio.unsqueeze(0) |
|
|
|
|
|
TACOTRON_MEL_MAX = 2.3143386840820312 |
|
TACOTRON_MEL_MIN = -11.512925148010254 |
|
|
|
|
|
def denormalize_tacotron_mel(norm_mel): |
|
return ((norm_mel+1)/2)*(TACOTRON_MEL_MAX-TACOTRON_MEL_MIN)+TACOTRON_MEL_MIN |
|
|
|
|
|
def normalize_tacotron_mel(mel): |
|
return 2 * ((mel - TACOTRON_MEL_MIN) / (TACOTRON_MEL_MAX - TACOTRON_MEL_MIN)) - 1 |
|
|
|
|
|
def dynamic_range_compression(x, C=1, clip_val=1e-5): |
|
""" |
|
PARAMS |
|
------ |
|
C: compression factor |
|
""" |
|
return torch.log(torch.clamp(x, min=clip_val) * C) |
|
|
|
|
|
def dynamic_range_decompression(x, C=1): |
|
""" |
|
PARAMS |
|
------ |
|
C: compression factor used to compress |
|
""" |
|
return torch.exp(x) / C |
|
|
|
|
|
def get_voices(): |
|
subs = os.listdir('voices') |
|
voices = {} |
|
for sub in subs: |
|
subj = os.path.join('voices', sub) |
|
if os.path.isdir(subj): |
|
voices[sub] = glob(f'{subj}/*.wav') |
|
return voices |
|
|
|
|
|
class TacotronSTFT(torch.nn.Module): |
|
def __init__(self, filter_length=1024, hop_length=256, win_length=1024, |
|
n_mel_channels=80, sampling_rate=22050, mel_fmin=0.0, |
|
mel_fmax=8000.0): |
|
super(TacotronSTFT, self).__init__() |
|
self.n_mel_channels = n_mel_channels |
|
self.sampling_rate = sampling_rate |
|
self.stft_fn = STFT(filter_length, hop_length, win_length) |
|
from librosa.filters import mel as librosa_mel_fn |
|
mel_basis = librosa_mel_fn( |
|
sampling_rate, filter_length, n_mel_channels, mel_fmin, mel_fmax) |
|
mel_basis = torch.from_numpy(mel_basis).float() |
|
self.register_buffer('mel_basis', mel_basis) |
|
|
|
def spectral_normalize(self, magnitudes): |
|
output = dynamic_range_compression(magnitudes) |
|
return output |
|
|
|
def spectral_de_normalize(self, magnitudes): |
|
output = dynamic_range_decompression(magnitudes) |
|
return output |
|
|
|
def mel_spectrogram(self, y): |
|
"""Computes mel-spectrograms from a batch of waves |
|
PARAMS |
|
------ |
|
y: Variable(torch.FloatTensor) with shape (B, T) in range [-1, 1] |
|
|
|
RETURNS |
|
------- |
|
mel_output: torch.FloatTensor of shape (B, n_mel_channels, T) |
|
""" |
|
assert(torch.min(y.data) >= -10) |
|
assert(torch.max(y.data) <= 10) |
|
y = torch.clip(y, min=-1, max=1) |
|
|
|
magnitudes, phases = self.stft_fn.transform(y) |
|
magnitudes = magnitudes.data |
|
mel_output = torch.matmul(self.mel_basis, magnitudes) |
|
mel_output = self.spectral_normalize(mel_output) |
|
return mel_output |
|
|
|
|
|
def wav_to_univnet_mel(wav, do_normalization=False): |
|
stft = TacotronSTFT(1024, 256, 1024, 100, 24000, 0, 12000) |
|
stft = stft.cuda() |
|
mel = stft.mel_spectrogram(wav) |
|
if do_normalization: |
|
mel = normalize_tacotron_mel(mel) |
|
return mel |
