Spaces:
Running
Running
import glob | |
import re | |
import librosa | |
import torch | |
import yaml | |
from sklearn.preprocessing import StandardScaler | |
from torch import nn | |
from modules.parallel_wavegan.models import ParallelWaveGANGenerator | |
from modules.parallel_wavegan.utils import read_hdf5 | |
from utils.hparams import hparams | |
from utils.pitch_utils import f0_to_coarse | |
from vocoders.base_vocoder import BaseVocoder, register_vocoder | |
import numpy as np | |
def load_pwg_model(config_path, checkpoint_path, stats_path): | |
# load config | |
with open(config_path) as f: | |
config = yaml.load(f, Loader=yaml.Loader) | |
# setup | |
if torch.cuda.is_available(): | |
device = torch.device("cuda") | |
else: | |
device = torch.device("cpu") | |
model = ParallelWaveGANGenerator(**config["generator_params"]) | |
ckpt_dict = torch.load(checkpoint_path, map_location="cpu") | |
if 'state_dict' not in ckpt_dict: # official vocoder | |
model.load_state_dict(torch.load(checkpoint_path, map_location="cpu")["model"]["generator"]) | |
scaler = StandardScaler() | |
if config["format"] == "hdf5": | |
scaler.mean_ = read_hdf5(stats_path, "mean") | |
scaler.scale_ = read_hdf5(stats_path, "scale") | |
elif config["format"] == "npy": | |
scaler.mean_ = np.load(stats_path)[0] | |
scaler.scale_ = np.load(stats_path)[1] | |
else: | |
raise ValueError("support only hdf5 or npy format.") | |
else: # custom PWG vocoder | |
fake_task = nn.Module() | |
fake_task.model_gen = model | |
fake_task.load_state_dict(torch.load(checkpoint_path, map_location="cpu")["state_dict"], strict=False) | |
scaler = None | |
model.remove_weight_norm() | |
model = model.eval().to(device) | |
print(f"| Loaded model parameters from {checkpoint_path}.") | |
print(f"| PWG device: {device}.") | |
return model, scaler, config, device | |
class PWG(BaseVocoder): | |
def __init__(self): | |
if hparams['vocoder_ckpt'] == '': # load LJSpeech PWG pretrained model | |
base_dir = 'wavegan_pretrained' | |
ckpts = glob.glob(f'{base_dir}/checkpoint-*steps.pkl') | |
ckpt = sorted(ckpts, key= | |
lambda x: int(re.findall(f'{base_dir}/checkpoint-(\d+)steps.pkl', x)[0]))[-1] | |
config_path = f'{base_dir}/config.yaml' | |
print('| load PWG: ', ckpt) | |
self.model, self.scaler, self.config, self.device = load_pwg_model( | |
config_path=config_path, | |
checkpoint_path=ckpt, | |
stats_path=f'{base_dir}/stats.h5', | |
) | |
else: | |
base_dir = hparams['vocoder_ckpt'] | |
print(base_dir) | |
config_path = f'{base_dir}/config.yaml' | |
ckpt = sorted(glob.glob(f'{base_dir}/model_ckpt_steps_*.ckpt'), key= | |
lambda x: int(re.findall(f'{base_dir}/model_ckpt_steps_(\d+).ckpt', x)[0]))[-1] | |
print('| load PWG: ', ckpt) | |
self.scaler = None | |
self.model, _, self.config, self.device = load_pwg_model( | |
config_path=config_path, | |
checkpoint_path=ckpt, | |
stats_path=f'{base_dir}/stats.h5', | |
) | |
def spec2wav(self, mel, **kwargs): | |
# start generation | |
config = self.config | |
device = self.device | |
pad_size = (config["generator_params"]["aux_context_window"], | |
config["generator_params"]["aux_context_window"]) | |
c = mel | |
if self.scaler is not None: | |
c = self.scaler.transform(c) | |
with torch.no_grad(): | |
z = torch.randn(1, 1, c.shape[0] * config["hop_size"]).to(device) | |
c = np.pad(c, (pad_size, (0, 0)), "edge") | |
c = torch.FloatTensor(c).unsqueeze(0).transpose(2, 1).to(device) | |
p = kwargs.get('f0') | |
if p is not None: | |
p = f0_to_coarse(p) | |
p = np.pad(p, (pad_size,), "edge") | |
p = torch.LongTensor(p[None, :]).to(device) | |
y = self.model(z, c, p).view(-1) | |
wav_out = y.cpu().numpy() | |
return wav_out | |
def wav2spec(wav_fn, return_linear=False): | |
from data_gen.tts.data_gen_utils import process_utterance | |
res = process_utterance( | |
wav_fn, fft_size=hparams['fft_size'], | |
hop_size=hparams['hop_size'], | |
win_length=hparams['win_size'], | |
num_mels=hparams['audio_num_mel_bins'], | |
fmin=hparams['fmin'], | |
fmax=hparams['fmax'], | |
sample_rate=hparams['audio_sample_rate'], | |
loud_norm=hparams['loud_norm'], | |
min_level_db=hparams['min_level_db'], | |
return_linear=return_linear, vocoder='pwg', eps=float(hparams.get('wav2spec_eps', 1e-10))) | |
if return_linear: | |
return res[0], res[1].T, res[2].T # [T, 80], [T, n_fft] | |
else: | |
return res[0], res[1].T | |
def wav2mfcc(wav_fn): | |
fft_size = hparams['fft_size'] | |
hop_size = hparams['hop_size'] | |
win_length = hparams['win_size'] | |
sample_rate = hparams['audio_sample_rate'] | |
wav, _ = librosa.core.load(wav_fn, sr=sample_rate) | |
mfcc = librosa.feature.mfcc(y=wav, sr=sample_rate, n_mfcc=13, | |
n_fft=fft_size, hop_length=hop_size, | |
win_length=win_length, pad_mode="constant", power=1.0) | |
mfcc_delta = librosa.feature.delta(mfcc, order=1) | |
mfcc_delta_delta = librosa.feature.delta(mfcc, order=2) | |
mfcc = np.concatenate([mfcc, mfcc_delta, mfcc_delta_delta]).T | |
return mfcc | |