import hashlib import json import logging import os import time from pathlib import Path import io import librosa import maad import numpy as np from inference import slicer import parselmouth import soundfile import torch import torchaudio from hubert import hubert_model import utils from models import SynthesizerTrn logging.getLogger('numba').setLevel(logging.WARNING) logging.getLogger('matplotlib').setLevel(logging.WARNING) def resize2d_f0(x, target_len): source = np.array(x) source[source < 0.001] = np.nan target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)), source) res = np.nan_to_num(target) return res def get_f0(x, p_len, f0_up_key=0): time_step = 160 / 16000 * 1000 f0_min = 50 f0_max = 1100 f0_mel_min = 1127 * np.log(1 + f0_min / 700) f0_mel_max = 1127 * np.log(1 + f0_max / 700) f0 = parselmouth.Sound(x, 16000).to_pitch_ac( time_step=time_step / 1000, voicing_threshold=0.6, pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency'] pad_size = (p_len - len(f0) + 1) // 2 if (pad_size > 0 or p_len - len(f0) - pad_size > 0): f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode='constant') f0 *= pow(2, f0_up_key / 12) f0_mel = 1127 * np.log(1 + f0 / 700) f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1 f0_mel[f0_mel <= 1] = 1 f0_mel[f0_mel > 255] = 255 f0_coarse = np.rint(f0_mel).astype(np.int) return f0_coarse, f0 def clean_pitch(input_pitch): num_nan = np.sum(input_pitch == 1) if num_nan / len(input_pitch) > 0.9: input_pitch[input_pitch != 1] = 1 return input_pitch def plt_pitch(input_pitch): input_pitch = input_pitch.astype(float) input_pitch[input_pitch == 1] = np.nan return input_pitch def f0_to_pitch(ff): f0_pitch = 69 + 12 * np.log2(ff / 440) return f0_pitch def fill_a_to_b(a, b): if len(a) < len(b): for _ in range(0, len(b) - len(a)): a.append(a[0]) def mkdir(paths: list): for path in paths: if not os.path.exists(path): os.mkdir(path) class VitsSvc(object): def __init__(self): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.SVCVITS = None self.hps = None self.speakers = None self.hubert_soft = utils.get_hubert_model() def set_device(self, device): self.device = torch.device(device) self.hubert_soft.to(self.device) if self.SVCVITS != None: self.SVCVITS.to(self.device) def loadCheckpoint(self, path): self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json") self.SVCVITS = SynthesizerTrn( self.hps.data.filter_length // 2 + 1, self.hps.train.segment_size // self.hps.data.hop_length, **self.hps.model) _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None) _ = self.SVCVITS.eval().to(self.device) self.speakers = self.hps.spk def get_units(self, source, sr): source = source.unsqueeze(0).to(self.device) with torch.inference_mode(): units = self.hubert_soft.units(source) return units def get_unit_pitch(self, in_path, tran): source, sr = torchaudio.load(in_path) source = torchaudio.functional.resample(source, sr, 16000) if len(source.shape) == 2 and source.shape[1] >= 2: source = torch.mean(source, dim=0).unsqueeze(0) soft = self.get_units(source, sr).squeeze(0).cpu().numpy() f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0] * 2, tran) return soft, f0 def infer(self, speaker_id, tran, raw_path): speaker_id = self.speakers[speaker_id] sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0) soft, pitch = self.get_unit_pitch(raw_path, tran) f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device) stn_tst = torch.FloatTensor(soft) with torch.no_grad(): x_tst = stn_tst.unsqueeze(0).to(self.device) x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2) audio = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0, 0].data.float() return audio, audio.shape[-1] def inference(self, srcaudio, chara, tran, slice_db): sampling_rate, audio = srcaudio audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32) if len(audio.shape) > 1: audio = librosa.to_mono(audio.transpose(1, 0)) if sampling_rate != 16000: audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000) soundfile.write("tmpwav.wav", audio, 16000, format="wav") chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db) audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks) audio = [] for (slice_tag, data) in audio_data: length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate)) raw_path = io.BytesIO() soundfile.write(raw_path, data, audio_sr, format="wav") raw_path.seek(0) if slice_tag: _audio = np.zeros(length) else: out_audio, out_sr = self.infer(chara, tran, raw_path) _audio = out_audio.cpu().numpy() audio.extend(list(_audio)) audio = (np.array(audio) * 32768.0).astype('int16') return (self.hps.data.sampling_rate, audio)