r3gm's picture
Demo uploaded with RVC options and more languages
84e44a0
raw
history blame
18.2 kB
import torch
from lib.infer_pack.models import (
SynthesizerTrnMs256NSFsid,
SynthesizerTrnMs256NSFsid_nono,
SynthesizerTrnMs768NSFsid,
SynthesizerTrnMs768NSFsid_nono,
)
from vc_infer_pipeline import VC
import traceback, pdb
from lib.audio import load_audio
import numpy as np
import os
from fairseq import checkpoint_utils
import soundfile as sf
from gtts import gTTS
import edge_tts
import asyncio
import nest_asyncio
# model load
def get_vc(sid, to_return_protect0, to_return_protect1):
global n_spk, tgt_sr, net_g, vc, cpt, version
if sid == "" or sid == []:
global hubert_model
if hubert_model is not None: # change model or not
print("clean_empty_cache")
del net_g, n_spk, vc, hubert_model, tgt_sr # ,cpt
hubert_model = net_g = n_spk = vc = hubert_model = tgt_sr = None
if torch.cuda.is_available():
torch.cuda.empty_cache()
### if clean
if_f0 = cpt.get("f0", 1)
version = cpt.get("version", "v1")
if version == "v1":
if if_f0 == 1:
net_g = SynthesizerTrnMs256NSFsid(
*cpt["config"], is_half=config.is_half
)
else:
net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"])
elif version == "v2":
if if_f0 == 1:
net_g = SynthesizerTrnMs768NSFsid(
*cpt["config"], is_half=config.is_half
)
else:
net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"])
del net_g, cpt
if torch.cuda.is_available():
torch.cuda.empty_cache()
return {"visible": False, "__type__": "update"}
person = "%s/%s" % (weight_root, sid)
print("loading %s" % person)
cpt = torch.load(person, map_location="cpu")
tgt_sr = cpt["config"][-1]
cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] # n_spk
if_f0 = cpt.get("f0", 1)
if if_f0 == 0:
to_return_protect0 = to_return_protect1 = {
"visible": False,
"value": 0.5,
"__type__": "update",
}
else:
to_return_protect0 = {
"visible": True,
"value": to_return_protect0,
"__type__": "update",
}
to_return_protect1 = {
"visible": True,
"value": to_return_protect1,
"__type__": "update",
}
version = cpt.get("version", "v1")
if version == "v1":
if if_f0 == 1:
net_g = SynthesizerTrnMs256NSFsid(*cpt["config"], is_half=config.is_half)
else:
net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"])
elif version == "v2":
if if_f0 == 1:
net_g = SynthesizerTrnMs768NSFsid(*cpt["config"], is_half=config.is_half)
else:
net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"])
del net_g.enc_q
print(net_g.load_state_dict(cpt["weight"], strict=False))
net_g.eval().to(config.device)
if config.is_half:
net_g = net_g.half()
else:
net_g = net_g.float()
vc = VC(tgt_sr, config)
n_spk = cpt["config"][-3]
return (
{"visible": True, "maximum": n_spk, "__type__": "update"},
to_return_protect0,
to_return_protect1,
)
# inference
def vc_single(
sid,
input_audio_path,
f0_up_key,
f0_file,
f0_method,
file_index,
file_index2,
# file_big_npy,
index_rate,
filter_radius,
resample_sr,
rms_mix_rate,
protect,
):
global tgt_sr, net_g, vc, hubert_model, version, cpt
if input_audio_path is None:
return "You need to upload an audio", None
f0_up_key = int(f0_up_key)
try:
audio = load_audio(input_audio_path, 16000)
audio_max = np.abs(audio).max() / 0.95
if audio_max > 1:
audio /= audio_max
times = [0, 0, 0]
if not hubert_model:
load_hubert()
if_f0 = cpt.get("f0", 1)
file_index = (
(
file_index.strip(" ")
.strip('"')
.strip("\n")
.strip('"')
.strip(" ")
.replace("trained", "added")
)
if file_index != ""
else file_index2
) # reemplace for 2
# file_big_npy = (
# file_big_npy.strip(" ").strip('"').strip("\n").strip('"').strip(" ")
# )
audio_opt = vc.pipeline(
hubert_model,
net_g,
sid,
audio,
input_audio_path,
times,
f0_up_key,
f0_method,
file_index,
# file_big_npy,
index_rate,
if_f0,
filter_radius,
tgt_sr,
resample_sr,
rms_mix_rate,
version,
protect,
f0_file=f0_file,
)
if tgt_sr != resample_sr >= 16000:
tgt_sr = resample_sr
index_info = (
"Using index:%s." % file_index
if os.path.exists(file_index)
else "Index not used."
)
return "Success.\n %s\nTime:\n npy:%ss, f0:%ss, infer:%ss" % (
index_info,
times[0],
times[1],
times[2],
), (tgt_sr, audio_opt)
except:
info = traceback.format_exc()
print(info)
return info, (None, None)
# hubert model
def load_hubert():
global hubert_model
models, _, _ = checkpoint_utils.load_model_ensemble_and_task(
["hubert_base.pt"],
suffix="",
)
hubert_model = models[0]
hubert_model = hubert_model.to(config.device)
if config.is_half:
hubert_model = hubert_model.half()
else:
hubert_model = hubert_model.float()
hubert_model.eval()
# config cpu
def use_fp32_config():
for config_file in [
"32k.json",
"40k.json",
"48k.json",
"48k_v2.json",
"32k_v2.json",
]:
with open(f"configs/{config_file}", "r") as f:
strr = f.read().replace("true", "false")
with open(f"configs/{config_file}", "w") as f:
f.write(strr)
# config device and torch type
class Config:
def __init__(self, device, is_half):
self.device = device
self.is_half = is_half
self.n_cpu = 2 # set cpu cores ####################
self.gpu_name = None
self.gpu_mem = None
self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config()
def device_config(self) -> tuple:
if torch.cuda.is_available():
i_device = int(self.device.split(":")[-1])
self.gpu_name = torch.cuda.get_device_name(i_device)
if (
("16" in self.gpu_name and "V100" not in self.gpu_name.upper())
or "P40" in self.gpu_name.upper()
or "1060" in self.gpu_name
or "1070" in self.gpu_name
or "1080" in self.gpu_name
):
print("16 series / 10 series graphics cards and P40 force single precision")
self.is_half = False
for config_file in ["32k.json", "40k.json", "48k.json"]:
with open(f"configs/{config_file}", "r") as f:
strr = f.read().replace("true", "false")
with open(f"configs/{config_file}", "w") as f:
f.write(strr)
with open("trainset_preprocess_pipeline_print.py", "r") as f:
strr = f.read().replace("3.7", "3.0")
with open("trainset_preprocess_pipeline_print.py", "w") as f:
f.write(strr)
else:
self.gpu_name = None
self.gpu_mem = int(
torch.cuda.get_device_properties(i_device).total_memory
/ 1024
/ 1024
/ 1024
+ 0.4
)
if self.gpu_mem <= 4:
with open("trainset_preprocess_pipeline_print.py", "r") as f:
strr = f.read().replace("3.7", "3.0")
with open("trainset_preprocess_pipeline_print.py", "w") as f:
f.write(strr)
elif torch.backends.mps.is_available():
print("Supported N-card not found, using MPS for inference")
self.device = "mps"
else:
print("No supported N-card found, using CPU for inference")
self.device = "cpu"
self.is_half = False
use_fp32_config()
if self.n_cpu == 0:
self.n_cpu = cpu_count()
if self.is_half:
# 6GB VRAM configuration
x_pad = 3
x_query = 10
x_center = 60
x_max = 65
else:
# 5GB VRAM configuration
x_pad = 1
x_query = 6
x_center = 38
x_max = 41
if self.gpu_mem != None and self.gpu_mem <= 4:
x_pad = 1
x_query = 5
x_center = 30
x_max = 32
print(self.device, self.is_half)
return x_pad, x_query, x_center, x_max
# call inference
class ClassVoices:
def __init__(self):
self.file_index = "" # root
def apply_conf(self, f0method,
model_voice_path00, transpose00, file_index2_00,
model_voice_path01, transpose01, file_index2_01,
model_voice_path02, transpose02, file_index2_02,
model_voice_path03, transpose03, file_index2_03,
model_voice_path04, transpose04, file_index2_04,
model_voice_path05, transpose05, file_index2_05,
model_voice_path99, transpose99, file_index2_99):
#self.filename = filename
self.f0method = f0method # pm
self.model_voice_path00 = model_voice_path00
self.transpose00 = transpose00
self.file_index200 = file_index2_00
self.model_voice_path01 = model_voice_path01
self.transpose01 = transpose01
self.file_index201 = file_index2_01
self.model_voice_path02 = model_voice_path02
self.transpose02 = transpose02
self.file_index202 = file_index2_02
self.model_voice_path03 = model_voice_path03
self.transpose03 = transpose03
self.file_index203 = file_index2_03
self.model_voice_path04 = model_voice_path04
self.transpose04 = transpose04
self.file_index204 = file_index2_04
self.model_voice_path05 = model_voice_path05
self.transpose05 = transpose05
self.file_index205 = file_index2_05
self.model_voice_path99 = model_voice_path99
self.transpose99 = transpose99
self.file_index299 = file_index2_99
return "CONFIGURATION APPLIED"
def custom_voice(self,
_values, # filter indices
audio_files, # all audio files
model_voice_path='',
transpose=0,
f0method='pm',
file_index='',
file_index2='',
):
#hubert_model = None
get_vc(
sid=model_voice_path, # model path
to_return_protect0=0.33,
to_return_protect1=0.33
)
for _value_item in _values:
filename = "audio2/"+audio_files[_value_item] if _value_item != "test" else audio_files[0]
#filename = "audio2/"+audio_files[_value_item]
try:
print(audio_files[_value_item], model_voice_path)
except:
pass
info_, (sample_, audio_output_) = vc_single(
sid=0,
input_audio_path=filename, #f"audio2/{filename}",
f0_up_key=transpose, # transpose for m to f and reverse 0 12
f0_file=None,
f0_method= f0method,
file_index= file_index, # dir pwd?
file_index2= file_index2,
# file_big_npy1,
index_rate= float(0.66),
filter_radius= int(3),
resample_sr= int(0),
rms_mix_rate= float(0.25),
protect= float(0.33),
)
sf.write(
file= filename, #f"audio2/{filename}",
samplerate=sample_,
data=audio_output_
)
# detele the model
def make_test(self,
tts_text,
tts_voice,
model_path,
index_path,
transpose,
f0_method,
):
os.system("rm -rf test")
filename = "test/test.wav"
if "SET_LIMIT" == os.getenv("DEMO"):
if len(tts_text) > 60:
tts_text = tts_text[:60]
print("DEMO; limit to 60 characters")
language = tts_voice[:2]
try:
os.system("mkdir test")
#nest_asyncio.apply() # gradio;not
asyncio.run(edge_tts.Communicate(tts_text, "-".join(tts_voice.split('-')[:-1])).save(filename))
except:
try:
tts = gTTS(tts_text, lang=language)
tts.save(filename)
tts.save
print(f'No audio was received. Please change the tts voice for {tts_voice}. USING gTTS.')
except:
tts = gTTS('a', lang=language)
tts.save(filename)
print('Error: Audio will be replaced.')
os.system("cp test/test.wav test/real_test.wav")
self([],[]) # start modules
self.custom_voice(
["test"], # filter indices
["test/test.wav"], # all audio files
model_voice_path=model_path,
transpose=transpose,
f0method=f0_method,
file_index='',
file_index2=index_path,
)
return "test/test.wav", "test/real_test.wav"
def __call__(self, speakers_list, audio_files):
speakers_indices = {}
for index, speak_ in enumerate(speakers_list):
if speak_ in speakers_indices:
speakers_indices[speak_].append(index)
else:
speakers_indices[speak_] = [index]
# find models and index
global weight_root, index_root, config, hubert_model
weight_root = "weights"
names = []
for name in os.listdir(weight_root):
if name.endswith(".pth"):
names.append(name)
index_root = "logs"
index_paths = []
for name in os.listdir(index_root):
if name.endswith(".index"):
index_paths.append(name)
print(names, index_paths)
# config machine
hubert_model = None
config = Config('cuda:0', is_half=True) # config = Config('cpu', is_half=False) # cpu
# filter by speaker
for _speak, _values in speakers_indices.items():
#print(_speak, _values)
#for _value_item in _values:
# self.filename = "audio2/"+audio_files[_value_item]
###print(audio_files[_value_item])
#vc(_speak, _values, audio_files)
if _speak == "SPEAKER_00":
self.custom_voice(
_values, # filteredd
audio_files,
model_voice_path=self.model_voice_path00,
file_index2=self.file_index200,
transpose=self.transpose00,
f0method=self.f0method,
file_index=self.file_index,
)
elif _speak == "SPEAKER_01":
self.custom_voice(
_values,
audio_files,
model_voice_path=self.model_voice_path01,
file_index2=self.file_index201,
transpose=self.transpose01,
f0method=self.f0method,
file_index=self.file_index,
)
elif _speak == "SPEAKER_02":
self.custom_voice(
_values,
audio_files,
model_voice_path=self.model_voice_path02,
file_index2=self.file_index202,
transpose=self.transpose02,
f0method=self.f0method,
file_index=self.file_index,
)
elif _speak == "SPEAKER_03":
self.custom_voice(
_values,
audio_files,
model_voice_path=self.model_voice_path03,
file_index2=self.file_index203,
transpose=self.transpose03,
f0method=self.f0method,
file_index=self.file_index,
)
elif _speak == "SPEAKER_04":
self.custom_voice(
_values,
audio_files,
model_voice_path=self.model_voice_path04,
file_index2=self.file_index204,
transpose=self.transpose04,
f0method=self.f0method,
file_index=self.file_index,
)
elif _speak == "SPEAKER_05":
self.custom_voice(
_values,
audio_files,
model_voice_path=self.model_voice_path05,
file_index2=self.file_index205,
transpose=self.transpose05,
f0method=self.f0method,
file_index=self.file_index,
)
elif _speak == "SPEAKER_99":
self.custom_voice(
_values,
audio_files,
model_voice_path=self.model_voice_path99,
file_index2=self.file_index299,
transpose=self.transpose99,
f0method=self.f0method,
file_index=self.file_index,
)
else:
pass