Spaces:
Running
Running
| import sys,os | |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| from vits.models import SynthesizerInfer | |
| from omegaconf import OmegaConf | |
| import torchcrepe | |
| import torch | |
| import io | |
| import os | |
| import gradio as gr | |
| import librosa | |
| import numpy as np | |
| import soundfile | |
| import logging | |
| logging.getLogger('numba').setLevel(logging.WARNING) | |
| logging.getLogger('markdown_it').setLevel(logging.WARNING) | |
| logging.getLogger('urllib3').setLevel(logging.WARNING) | |
| logging.getLogger('matplotlib').setLevel(logging.WARNING) | |
| def load_svc_model(checkpoint_path, model): | |
| assert os.path.isfile(checkpoint_path) | |
| checkpoint_dict = torch.load(checkpoint_path, map_location="cpu") | |
| saved_state_dict = checkpoint_dict["model_g"] | |
| state_dict = model.state_dict() | |
| new_state_dict = {} | |
| for k, v in state_dict.items(): | |
| new_state_dict[k] = saved_state_dict[k] | |
| model.load_state_dict(new_state_dict) | |
| return model | |
| def compute_f0_nn(filename, device): | |
| audio, sr = librosa.load(filename, sr=16000) | |
| assert sr == 16000 | |
| # Load audio | |
| audio = torch.tensor(np.copy(audio))[None] | |
| # Here we'll use a 20 millisecond hop length | |
| hop_length = 320 | |
| # Provide a sensible frequency range for your domain (upper limit is 2006 Hz) | |
| # This would be a reasonable range for speech | |
| fmin = 50 | |
| fmax = 1000 | |
| # Select a model capacity--one of "tiny" or "full" | |
| model = "full" | |
| # Pick a batch size that doesn't cause memory errors on your gpu | |
| batch_size = 512 | |
| # Compute pitch using first gpu | |
| pitch, periodicity = torchcrepe.predict( | |
| audio, | |
| sr, | |
| hop_length, | |
| fmin, | |
| fmax, | |
| model, | |
| batch_size=batch_size, | |
| device=device, | |
| return_periodicity=True, | |
| ) | |
| pitch = np.repeat(pitch, 2, -1) # 320 -> 160 * 2 | |
| periodicity = np.repeat(periodicity, 2, -1) # 320 -> 160 * 2 | |
| # CREPE was not trained on silent audio. some error on silent need filter. | |
| periodicity = torchcrepe.filter.median(periodicity, 9) | |
| pitch = torchcrepe.filter.mean(pitch, 3) | |
| # pitch[periodicity < 0.1] = 0 | |
| pitch = pitch.squeeze(0) | |
| return pitch | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| hp = OmegaConf.load("configs/base.yaml") | |
| model = SynthesizerInfer( | |
| hp.data.filter_length // 2 + 1, | |
| hp.data.segment_size // hp.data.hop_length, | |
| hp) | |
| load_svc_model("vits_pretrain/sovits5.0_bigvgan_mix.pth", model) | |
| model.eval() | |
| model.to(device) | |
| def svc_change(argswave, argsspk): | |
| argsppg = "svc_tmp.ppg.npy" | |
| os.system(f"python whisper/inference.py -w {argswave} -p {argsppg}") | |
| argsvec = "svc_tmp.vec.npy" | |
| os.system(f"python hubert/inference.py -w {argswave} -v {argsvec}") | |
| spk = np.load(argsspk) | |
| spk = torch.FloatTensor(spk) | |
| ppg = np.load(argsppg) | |
| ppg = np.repeat(ppg, 2, 0) # 320 PPG -> 160 * 2 | |
| ppg = torch.FloatTensor(ppg) | |
| vec = np.load(argsvec) | |
| vec = np.repeat(vec, 2, 0) # 320 PPG -> 160 * 2 | |
| vec = torch.FloatTensor(vec) | |
| pit = compute_f0_nn(argswave, device) | |
| pit = torch.FloatTensor(pit) | |
| len_pit = pit.size()[0] | |
| len_vec = vec.size()[0] | |
| len_ppg = ppg.size()[0] | |
| len_min = min(len_pit, len_vec) | |
| len_min = min(len_min, len_ppg) | |
| pit = pit[:len_min] | |
| vec = vec[:len_min, :] | |
| ppg = ppg[:len_min, :] | |
| with torch.no_grad(): | |
| spk = spk.unsqueeze(0).to(device) | |
| source = pit.unsqueeze(0).to(device) | |
| source = model.pitch2source(source) | |
| hop_size = hp.data.hop_length | |
| all_frame = len_min | |
| hop_frame = 10 | |
| out_chunk = 2500 # 25 S | |
| out_index = 0 | |
| out_audio = [] | |
| has_audio = False | |
| while (out_index + out_chunk < all_frame): | |
| has_audio = True | |
| if (out_index == 0): # start frame | |
| cut_s = out_index | |
| cut_s_out = 0 | |
| else: | |
| cut_s = out_index - hop_frame | |
| cut_s_out = hop_frame * hop_size | |
| if (out_index + out_chunk + hop_frame > all_frame): # end frame | |
| cut_e = out_index + out_chunk | |
| cut_e_out = 0 | |
| else: | |
| cut_e = out_index + out_chunk + hop_frame | |
| cut_e_out = -1 * hop_frame * hop_size | |
| sub_ppg = ppg[cut_s:cut_e, :].unsqueeze(0).to(device) | |
| sub_vec = vec[cut_s:cut_e, :].unsqueeze(0).to(device) | |
| sub_pit = pit[cut_s:cut_e].unsqueeze(0).to(device) | |
| sub_len = torch.LongTensor([cut_e - cut_s]).to(device) | |
| sub_har = source[:, :, cut_s * | |
| hop_size:cut_e * hop_size].to(device) | |
| sub_out = model.inference(sub_ppg, sub_vec, sub_pit, spk, sub_len, sub_har) | |
| sub_out = sub_out[0, 0].data.cpu().detach().numpy() | |
| sub_out = sub_out[cut_s_out:cut_e_out] | |
| out_audio.extend(sub_out) | |
| out_index = out_index + out_chunk | |
| if (out_index < all_frame): | |
| if (has_audio): | |
| cut_s = out_index - hop_frame | |
| cut_s_out = hop_frame * hop_size | |
| else: | |
| cut_s = 0 | |
| cut_s_out = 0 | |
| sub_ppg = ppg[cut_s:, :].unsqueeze(0).to(device) | |
| sub_vec = vec[cut_s:, :].unsqueeze(0).to(device) | |
| sub_pit = pit[cut_s:].unsqueeze(0).to(device) | |
| sub_len = torch.LongTensor([all_frame - cut_s]).to(device) | |
| sub_har = source[:, :, cut_s * hop_size:].to(device) | |
| sub_out = model.inference(sub_ppg, sub_vec, sub_pit, spk, sub_len, sub_har) | |
| sub_out = sub_out[0, 0].data.cpu().detach().numpy() | |
| sub_out = sub_out[cut_s_out:] | |
| out_audio.extend(sub_out) | |
| out_audio = np.asarray(out_audio) | |
| return out_audio | |
| def svc_main(sid, input_audio): | |
| if input_audio is None: | |
| return "You need to upload an audio", None | |
| sampling_rate, audio = input_audio | |
| audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32) | |
| if len(audio.shape) > 1: | |
| audio = librosa.to_mono(audio.transpose(1, 0)) | |
| if sampling_rate != 16000: | |
| audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000) | |
| if (len(audio) > 16000*100): | |
| audio = audio[:16000*100] | |
| wav_path = "temp.wav" | |
| soundfile.write(wav_path, audio, 16000, format="wav") | |
| out_audio = svc_change(wav_path, f"configs/singers/singer00{sid}.npy") | |
| return "Success", (32000, out_audio) | |
| app = gr.Blocks() | |
| with app: | |
| with gr.Tabs(): | |
| with gr.TabItem("sovits 5.0"): | |
| gr.Markdown(value=""" | |
| 基于开源数据:Multi-Singer | |
| https://github.com/Multi-Singer/Multi-Singer.github.io | |
| 最终版本： | |
| 1，mix_encoder: whisper + hubert, 解决跨语言转换和纯对白语音训练 | |
| 2，解决F0瑕疵 | |
| """) | |
| sid = gr.Dropdown(label="音色", choices=[ | |
| "22", "33", "47", "51"], value="47") | |
| vc_input3 = gr.Audio(label="上传音频") | |
| vc_submit = gr.Button("转换", variant="primary") | |
| vc_output1 = gr.Textbox(label="状态信息") | |
| vc_output2 = gr.Audio(label="转换音频") | |
| vc_submit.click(svc_main, [sid, vc_input3], [vc_output1, vc_output2]) | |
| app.launch() | |