Spaces:
Sleeping
Sleeping
| import argparse | |
| import os | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tempfile | |
| from functools import partial | |
| import gradio as gr | |
| import librosa | |
| import numpy as np | |
| import soundfile | |
| import torch | |
| from edgetts.tts_voices import SUPPORTED_LANGUAGES | |
| from inference.infer_tool import Svc | |
| MAXOCTAVE = 2 | |
| TEMPDIR = None | |
| def generate_tempfile(suffix=None, prefix=None): | |
| global TEMPDIR | |
| _, filepath = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=TEMPDIR) | |
| return filepath | |
| def find_sovits_model(dirpath): | |
| for filename in os.listdir(dirpath): | |
| if filename.endswith(".pth"): | |
| return os.path.join(dirpath, filename) | |
| return None | |
| def find_diffusion_model(dirpath): | |
| for filename in os.listdir(dirpath): | |
| if filename.startswith("model") and filename.endswith(".pt"): | |
| return os.path.join(dirpath, filename) | |
| return None | |
| def find_static_file(dirpath, filename): | |
| filepath = os.path.join(dirpath, filename) | |
| return filepath if os.path.exists(filepath) else None | |
| def model_fn(modeldir, model, leakctrl, diffonly, enhancer): | |
| if model is not None: | |
| model.unload_model() | |
| # locate trained models | |
| sovits_model_path = find_sovits_model(modeldir) | |
| sovits_config_path = find_static_file(modeldir, "config.json") | |
| diffusion_model_path = find_diffusion_model(modeldir) | |
| diffusion_config_path = find_static_file(modeldir, "config.yaml") | |
| kmeans_model_path = find_static_file(modeldir, "kmeans_10000.pt") | |
| feature_index_path = find_static_file(modeldir, "feature_and_index.pkl") | |
| feature_retrieval = leakctrl == "Feature retrieval" | |
| cluster_model_path = feature_index_path if feature_retrieval else kmeans_model_path | |
| model = Svc( | |
| sovits_model_path, | |
| sovits_config_path, | |
| cluster_model_path=cluster_model_path, | |
| feature_retrieval=feature_retrieval, | |
| diffusion_model_path=diffusion_model_path, | |
| diffusion_config_path=diffusion_config_path, | |
| shallow_diffusion=True, | |
| only_diffusion=diffonly, | |
| nsf_hifigan_enhance=enhancer, | |
| ) | |
| speakers = list(model.spk2id.keys()) | |
| return ( | |
| model, | |
| "Reload Model", | |
| f"Successfully loaded model into device {str(model.dev)}", | |
| gr.Dropdown(choices=speakers, value=speakers[0]), | |
| ) | |
| def preset_fn(preset): | |
| if preset == "Singing": | |
| f0_predictor = "none" | |
| leakctrl_ratio = 0.5 | |
| else: | |
| f0_predictor = "rmvpe" | |
| leakctrl_ratio = 0 | |
| """ | |
| f0_predictor, pitch_shift, leakctrl_ratio, diff_steps, noise_scale, | |
| silent_padding, db_threshold, auto_clip, clip_overlap, cross_fade, | |
| adaptive_key, crepe_f0, loudness_ratio, reencode_audio, | |
| """ | |
| return ( | |
| f0_predictor, 0, leakctrl_ratio, 100, 0.4, | |
| 0.5, -40, 0, 0, 0.75, | |
| 0, 0.05, 0, False, | |
| ) | |
| def tts_fn(text, gender, lang, rate, volume): | |
| def to_percent(x): | |
| return f"+{int(x * 100)}%" if x >= 0 else f"{int(x * 100)}%" | |
| rate = to_percent(rate) | |
| volume = to_percent(volume) | |
| outfile = generate_tempfile(suffix=".wav") | |
| subprocess.run([sys.executable, "edgetts/tts.py", text, lang, rate, volume, gender, outfile]) | |
| result, orig_sr = librosa.load(outfile) | |
| os.remove(outfile) | |
| target_sr = 44100 | |
| resampled = librosa.resample(result, orig_sr=orig_sr, target_sr=target_sr) | |
| return target_sr, resampled | |
| def inference_fn( | |
| model, speaker, input_audio, | |
| f0_predictor, pitch_shift, leakctrl_ratio, diff_steps, noise_scale, | |
| silent_padding, db_threshold, auto_clip, clip_overlap, cross_fade, | |
| adaptive_key, crepe_f0, loudness_ratio, reencode_audio, | |
| ): | |
| if model is None: | |
| return "Error: please load model first", None | |
| if input_audio is None: | |
| return "Error: please upload an audio", None | |
| sample_rate, audio = input_audio | |
| if np.issubdtype(audio.dtype, np.integer): | |
| audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32) | |
| if len(audio.shape) > 1: | |
| audio = librosa.to_mono(audio.transpose(1, 0)) | |
| infile = generate_tempfile(suffix=".wav") | |
| soundfile.write(infile, audio, sample_rate, format="wav") | |
| result = model.slice_inference( | |
| infile, | |
| speaker, | |
| pitch_shift, | |
| db_threshold, | |
| leakctrl_ratio, | |
| f0_predictor != "none", | |
| noise_scale, | |
| pad_seconds=silent_padding, | |
| clip_seconds=auto_clip, | |
| lg_num=clip_overlap, | |
| lgr_num=cross_fade, | |
| f0_predictor="crepe" if f0_predictor == "none" else f0_predictor, | |
| enhancer_adaptive_key=adaptive_key, | |
| cr_threshold=crepe_f0, | |
| k_step=diff_steps, | |
| use_spk_mix=False, | |
| second_encoding=reencode_audio, | |
| loudness_envelope_adjustment=loudness_ratio, | |
| ) | |
| model.clear_empty() | |
| os.remove(infile) | |
| # gr.Audio force normalize the audio if supplied as a numpy array | |
| # we must write to a temporary file and return the filepath here | |
| prefix = f"{speaker}_{f0_predictor}_pitch{pitch_shift}_timbre{leakctrl_ratio}_diff{diff_steps}_" | |
| outfile = generate_tempfile(suffix=".wav", prefix=prefix) | |
| soundfile.write(outfile, result, model.target_sample, format="wav") | |
| return "Success", outfile | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser(description="so-vits-svc WebUI") | |
| parser.add_argument("-m", "--model", default="./trained") | |
| parser.add_argument("-t", "--temp", default="./workspace") | |
| args = parser.parse_args() | |
| print(f"Is CUDA available: {torch.cuda.is_available()}") | |
| if torch.cuda.is_available(): | |
| print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}") | |
| shutil.rmtree(args.temp, ignore_errors=True) | |
| os.makedirs(args.temp, exist_ok=True) | |
| TEMPDIR = args.temp | |
| with gr.Blocks() as app: | |
| with gr.Row(): | |
| with gr.Column(): | |
| title = gr.Markdown(value="""# AI Sora Singing Voice Conversion""") | |
| with gr.Column(): | |
| with gr.Accordion(label="About", open=False): | |
| about = gr.Markdown(value="""Space by [KasugaiSakura](https://huggingface.co/KasugaiSakura)<br/>Based on a modified version of [so-vits-svc](https://github.com/meimisaki/so-vits-svc/tree/4.1-Stable)<br/>Voice copyright belongs to [CUFFS/Sphere](https://www.cuffs.co.jp/)""") | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Accordion(label="Model setup", open=True): | |
| leakctrl = gr.Radio( | |
| label="Timbre leakage control method", | |
| choices=["Feature retrieval", "K-means clustering"], | |
| value="Feature retrieval", | |
| ) | |
| diffonly = gr.Checkbox(label="Diffusion only mode") | |
| enhancer = gr.Checkbox(label="NSF-HiFiGAN enhancer (not recommended)") | |
| modelptr = gr.State(None) | |
| modelbtn = gr.Button(value="Load Model", variant="primary") | |
| modelmsg = gr.Textbox(label="Model info") | |
| speaker = gr.Dropdown(label="Speaker", interactive=True) | |
| with gr.Accordion(label="Text to speech", open=False): | |
| tts_text = gr.Textbox(label="Text", placeholder="Enter text here") | |
| tts_gender = gr.Radio(label="Gender", choices=["Male","Female"], value="Male") | |
| tts_lang = gr.Dropdown(label="Language", choices=SUPPORTED_LANGUAGES, value="Auto") | |
| tts_rate = gr.Slider( | |
| label="Relative speed", | |
| minimum=-1, maximum=3, value=0, step=0.1 | |
| ) | |
| tts_volume = gr.Slider( | |
| label="Relative volume", | |
| minimum=-1, maximum=1.5, value=0, step=0.1 | |
| ) | |
| tts_btn = gr.Button(value="Synthesize") | |
| with gr.Accordion(label="Voice conversion", open=True): | |
| input_audio = gr.Audio(label="Input audio", type="numpy") | |
| inference_btn = gr.Button(value="Inference") | |
| output_msg = gr.Textbox(label="Output message") | |
| output_audio = gr.Audio(label="Output audio", type="filepath") | |
| with gr.Column(): | |
| with gr.Accordion(label="Inference options", open=True): | |
| inference_preset = gr.Radio( | |
| label="Preset", | |
| choices=["Singing", "Speaking"], | |
| value="Singing", | |
| interactive=True, | |
| ) | |
| f0_predictor = gr.Dropdown( | |
| label="F0 predictor", | |
| choices=["none", "crepe", "dio", "harvest", "pm", "rmvpe"], | |
| value="none", | |
| ) | |
| pitch_shift = gr.Slider( | |
| label="Pitch shift (in semitones, 12 in an octave)", | |
| minimum=-12*MAXOCTAVE, maximum=12*MAXOCTAVE, value=0, step=1, | |
| ) | |
| leakctrl_ratio = gr.Slider( | |
| label="Timbre leakage control mix ratio (set to 0 to disable it)", | |
| minimum=0, maximum=1, value=0.5, step=0.1, | |
| ) | |
| diff_steps = gr.Slider( | |
| label="Shallow diffusion steps", | |
| minimum=0, maximum=1000, value=100, step=10, | |
| ) | |
| noise_scale = gr.Slider( | |
| label="Noise scale (try NOT to modify this parameter)", | |
| minimum=0, maximum=1, value=0.4, step=0.01, | |
| ) | |
| silent_padding = gr.Slider( | |
| label="Add silent padding to workaround noise caused by unknown reason (in seconds)", | |
| minimum=0, maximum=3, value=0.5, step=0.01, | |
| ) | |
| db_threshold = gr.Slider( | |
| label="Silence dB threshold (for slicing audio into chunks)", | |
| minimum=-100, maximum=0, value=-40, step=1, | |
| ) | |
| auto_clip = gr.Slider( | |
| label="Apply auto clip to reduce memory consumption (in seconds)", | |
| minimum=0, maximum=100, value=0, step=1, | |
| ) | |
| clip_overlap = gr.Slider( | |
| label="Overlap duration between auto clips (in seconds)", | |
| minimum=0, maximum=3, value=0, step=0.01, | |
| ) | |
| cross_fade = gr.Slider( | |
| label="Cross fade ratio of overlapping regions", | |
| minimum=0, maximum=1, value=0.75, step=0.01, | |
| ) | |
| adaptive_key = gr.Slider( | |
| label="Enhancer adaptive key (in semitones, 12 in an octave)", | |
| minimum=-12*MAXOCTAVE, maximum=12*MAXOCTAVE, value=0, step=1, | |
| ) | |
| crepe_f0 = gr.Slider( | |
| label="CREPE F0 threshold (increase to reduce noise but may result in out-of-tune)", | |
| minimum=0, maximum=1, value=0.05, step=0.01, | |
| ) | |
| loudness_ratio = gr.Slider( | |
| label="Loudness envelope mix ratio of input and output (0 is input and 1 is output)", | |
| minimum=0, maximum=1, value=0, step=0.01, | |
| ) | |
| reencode_audio = gr.Checkbox( | |
| label="Re-encode audio before shallow diffusion, with unknown impact on final result" | |
| ) | |
| modelbtn.click( | |
| partial(model_fn, args.model), | |
| inputs=[modelptr, leakctrl, diffonly, enhancer], | |
| outputs=[modelptr, modelbtn, modelmsg, speaker], | |
| ) | |
| inference_preset.change( | |
| preset_fn, | |
| inputs=[inference_preset], | |
| outputs=[ | |
| f0_predictor, pitch_shift, leakctrl_ratio, diff_steps, noise_scale, | |
| silent_padding, db_threshold, auto_clip, clip_overlap, cross_fade, | |
| adaptive_key, crepe_f0, loudness_ratio, reencode_audio, | |
| ], | |
| ) | |
| tts_btn.click( | |
| tts_fn, | |
| inputs=[tts_text, tts_gender, tts_lang, tts_rate, tts_volume], | |
| outputs=[input_audio], | |
| ) | |
| inference_btn.click( | |
| inference_fn, | |
| inputs=[ | |
| modelptr, speaker, input_audio, | |
| f0_predictor, pitch_shift, leakctrl_ratio, diff_steps, noise_scale, | |
| silent_padding, db_threshold, auto_clip, clip_overlap, cross_fade, | |
| adaptive_key, crepe_f0, loudness_ratio, reencode_audio, | |
| ], | |
| outputs=[output_msg, output_audio], | |
| ) | |
| app.launch(debug=True, share=True) | |