Spaces:
Runtime error
Runtime error
| import asyncio | |
| import datetime | |
| import logging | |
| import os | |
| import time | |
| import traceback | |
| import edge_tts | |
| import gradio as gr | |
| import librosa | |
| import torch | |
| from fairseq import checkpoint_utils | |
| from config import Config | |
| from lib.infer_pack.models import ( | |
| SynthesizerTrnMs256NSFsid, | |
| SynthesizerTrnMs256NSFsid_nono, | |
| SynthesizerTrnMs768NSFsid, | |
| SynthesizerTrnMs768NSFsid_nono, | |
| ) | |
| from rmvpe import RMVPE | |
| from vc_infer_pipeline import VC | |
| logging.getLogger("fairseq").setLevel(logging.WARNING) | |
| logging.getLogger("numba").setLevel(logging.WARNING) | |
| logging.getLogger("markdown_it").setLevel(logging.WARNING) | |
| logging.getLogger("urllib3").setLevel(logging.WARNING) | |
| logging.getLogger("matplotlib").setLevel(logging.WARNING) | |
| limitation = os.getenv("SYSTEM") == "spaces" | |
| config = Config() | |
| # Edge TTS | |
| edge_output_filename = "edge_output.mp3" | |
| tts_voice_list = asyncio.get_event_loop().run_until_complete(edge_tts.list_voices()) | |
| tts_voices = [f"{v['ShortName']}-{v['Gender']}" for v in tts_voice_list] | |
| # RVC models | |
| model_root = "weights" | |
| models = [d for d in os.listdir(model_root) if os.path.isdir(f"{model_root}/{d}")] | |
| models.sort() | |
| def model_data(model_name): | |
| # global n_spk, tgt_sr, net_g, vc, cpt, version, index_file | |
| pth_path = [ | |
| f"{model_root}/{model_name}/{f}" | |
| for f in os.listdir(f"{model_root}/{model_name}") | |
| if f.endswith(".pth") | |
| ][0] | |
| print(f"Loading {pth_path}") | |
| cpt = torch.load(pth_path, map_location="cpu") | |
| tgt_sr = cpt["config"][-1] | |
| cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] # n_spk | |
| if_f0 = cpt.get("f0", 1) | |
| version = cpt.get("version", "v1") | |
| if version == "v1": | |
| if if_f0 == 1: | |
| net_g = SynthesizerTrnMs256NSFsid(*cpt["config"], is_half=config.is_half) | |
| else: | |
| net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"]) | |
| elif version == "v2": | |
| if if_f0 == 1: | |
| net_g = SynthesizerTrnMs768NSFsid(*cpt["config"], is_half=config.is_half) | |
| else: | |
| net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"]) | |
| else: | |
| raise ValueError("Unknown version") | |
| del net_g.enc_q | |
| net_g.load_state_dict(cpt["weight"], strict=False) | |
| print("Model loaded") | |
| net_g.eval().to(config.device) | |
| if config.is_half: | |
| net_g = net_g.half() | |
| else: | |
| net_g = net_g.float() | |
| vc = VC(tgt_sr, config) | |
| # n_spk = cpt["config"][-3] | |
| index_files = [ | |
| f"{model_root}/{model_name}/{f}" | |
| for f in os.listdir(f"{model_root}/{model_name}") | |
| if f.endswith(".index") | |
| ] | |
| if len(index_files) == 0: | |
| print("No index file found") | |
| index_file = "" | |
| else: | |
| index_file = index_files[0] | |
| print(f"Index file found: {index_file}") | |
| return tgt_sr, net_g, vc, version, index_file, if_f0 | |
| def load_hubert(): | |
| # global hubert_model | |
| models, _, _ = checkpoint_utils.load_model_ensemble_and_task( | |
| ["hubert_base.pt"], | |
| suffix="", | |
| ) | |
| hubert_model = models[0] | |
| hubert_model = hubert_model.to(config.device) | |
| if config.is_half: | |
| hubert_model = hubert_model.half() | |
| else: | |
| hubert_model = hubert_model.float() | |
| return hubert_model.eval() | |
| def tts( | |
| model_name, | |
| speed, | |
| tts_text, | |
| tts_voice, | |
| f0_up_key, | |
| f0_method, | |
| index_rate, | |
| protect, | |
| filter_radius=3, | |
| resample_sr=0, | |
| rms_mix_rate=0.25, | |
| ): | |
| print("------------------") | |
| print(datetime.datetime.now()) | |
| print("tts_text:") | |
| print(tts_text) | |
| print(f"tts_voice: {tts_voice}, speed: {speed}") | |
| print(f"Model name: {model_name}") | |
| print(f"F0: {f0_method}, Key: {f0_up_key}, Index: {index_rate}, Protect: {protect}") | |
| try: | |
| if limitation and len(tts_text) > 280: | |
| print("Error: Text too long") | |
| return ( | |
| f"Text characters should be at most 280 in this huggingface space, but got {len(tts_text)} characters.", | |
| None, | |
| None, | |
| ) | |
| t0 = time.time() | |
| if speed >= 0: | |
| speed_str = f"+{speed}%" | |
| else: | |
| speed_str = f"{speed}%" | |
| asyncio.run( | |
| edge_tts.Communicate( | |
| tts_text, "-".join(tts_voice.split("-")[:-1]), rate=speed_str | |
| ).save(edge_output_filename) | |
| ) | |
| t1 = time.time() | |
| edge_time = t1 - t0 | |
| audio, sr = librosa.load(edge_output_filename, sr=16000, mono=True) | |
| duration = len(audio) / sr | |
| print(f"Audio duration: {duration}s") | |
| if limitation and duration >= 20: | |
| print("Error: Audio too long") | |
| return ( | |
| f"Audio should be less than 20 seconds in this huggingface space, but got {duration}s.", | |
| edge_output_filename, | |
| None, | |
| ) | |
| f0_up_key = int(f0_up_key) | |
| tgt_sr, net_g, vc, version, index_file, if_f0 = model_data(model_name) | |
| if f0_method == "rmvpe": | |
| vc.model_rmvpe = rmvpe_model | |
| times = [0, 0, 0] | |
| audio_opt = vc.pipeline( | |
| hubert_model, | |
| net_g, | |
| 0, | |
| audio, | |
| edge_output_filename, | |
| times, | |
| f0_up_key, | |
| f0_method, | |
| index_file, | |
| # file_big_npy, | |
| index_rate, | |
| if_f0, | |
| filter_radius, | |
| tgt_sr, | |
| resample_sr, | |
| rms_mix_rate, | |
| version, | |
| protect, | |
| None, | |
| ) | |
| if tgt_sr != resample_sr >= 16000: | |
| tgt_sr = resample_sr | |
| info = f"Success. Time: edge-tts: {edge_time}s, npy: {times[0]}s, f0: {times[1]}s, infer: {times[2]}s" | |
| print(info) | |
| return ( | |
| info, | |
| edge_output_filename, | |
| (tgt_sr, audio_opt), | |
| ) | |
| except EOFError: | |
| info = ( | |
| "It seems that the edge-tts output is not valid. " | |
| "This may occur when the input text and the speaker do not match. " | |
| "For example, maybe you entered Japanese (without alphabets) text but chose non-Japanese speaker?" | |
| ) | |
| print(info) | |
| return info, None, None | |
| except: | |
| info = traceback.format_exc() | |
| print(info) | |
| return info, None, None | |
| print("Loading hubert model...") | |
| hubert_model = load_hubert() | |
| print("Hubert model loaded.") | |
| print("Loading rmvpe model...") | |
| rmvpe_model = RMVPE("rmvpe.pt", config.is_half, config.device) | |
| print("rmvpe model loaded.") | |
| initial_md = """ | |
| # RVC text-to-speech demo | |
| This is a text-to-speech demo of RVC moe models of [rvc_okiba](https://huggingface.co/litagin/rvc_okiba) using [edge-tts](https://github.com/rany2/edge-tts). | |
| Input text ➡[(edge-tts)](https://github.com/rany2/edge-tts)➡ Speech mp3 file ➡[(RVC)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI)➡ Final output | |
| This runs on the 🤗 server's cpu, so it may be slow. | |
| Although the models are trained on Japanese voices and intended for Japanese text, they can also be used with other languages with the corresponding edge-tts speaker (but possibly with a Japanese accent). | |
| Input characters are limited to 280 characters, and the speech audio is limited to 20 seconds in this 🤗 space. | |
| [Visit this GitHub repo](https://github.com/litagin02/rvc-tts-webui) for running locally with your models and GPU! | |
| """ | |
| app = gr.Blocks() | |
| with app: | |
| gr.Markdown(initial_md) | |
| with gr.Row(): | |
| with gr.Column(): | |
| model_name = gr.Dropdown( | |
| label="Model (all models except man-_ are girl models)", | |
| choices=models, | |
| value=models[0], | |
| ) | |
| f0_key_up = gr.Number( | |
| label="Tune (+12 = 1 octave up from edge-tts, the best value depends on the models and speakers)", | |
| value=2, | |
| ) | |
| with gr.Column(): | |
| f0_method = gr.Radio( | |
| label="Pitch extraction method (pm: very fast, low quality, rmvpe: a little slow, high quality)", | |
| choices=["pm", "rmvpe"], # harvest and crepe is too slow | |
| value="rmvpe", | |
| interactive=True, | |
| ) | |
| index_rate = gr.Slider( | |
| minimum=0, | |
| maximum=1, | |
| label="Index rate", | |
| value=1, | |
| interactive=True, | |
| ) | |
| protect0 = gr.Slider( | |
| minimum=0, | |
| maximum=0.5, | |
| label="Protect", | |
| value=0.33, | |
| step=0.01, | |
| interactive=True, | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| tts_voice = gr.Dropdown( | |
| label="Edge-tts speaker (format: language-Country-Name-Gender), make sure the gender matches the model", | |
| choices=tts_voices, | |
| allow_custom_value=False, | |
| value="ja-JP-NanamiNeural-Female", | |
| ) | |
| speed = gr.Slider( | |
| minimum=-100, | |
| maximum=100, | |
| label="Speech speed (%)", | |
| value=0, | |
| step=10, | |
| interactive=True, | |
| ) | |
| tts_text = gr.Textbox(label="Input Text", value="これは日本語テキストから音声への変換デモです。") | |
| with gr.Column(): | |
| but0 = gr.Button("Convert", variant="primary") | |
| info_text = gr.Textbox(label="Output info") | |
| with gr.Column(): | |
| edge_tts_output = gr.Audio(label="Edge Voice", type="filepath") | |
| tts_output = gr.Audio(label="Result") | |
| but0.click( | |
| tts, | |
| [ | |
| model_name, | |
| speed, | |
| tts_text, | |
| tts_voice, | |
| f0_key_up, | |
| f0_method, | |
| index_rate, | |
| protect0, | |
| ], | |
| [info_text, edge_tts_output, tts_output], | |
| ) | |
| with gr.Row(): | |
| examples = gr.Examples( | |
| examples_per_page=100, | |
| examples=[ | |
| ["これは日本語テキストから音声への変換デモです。", "ja-JP-NanamiNeural-Female"], | |
| [ | |
| "This is an English text to speech conversation demo.", | |
| "en-US-AriaNeural-Female", | |
| ], | |
| ["这是一个中文文本到语音的转换演示。", "zh-CN-XiaoxiaoNeural-Female"], | |
| ["한국어 텍스트에서 음성으로 변환하는 데모입니다.", "ko-KR-SunHiNeural-Female"], | |
| [ | |
| "Il s'agit d'une démo de conversion du texte français à la parole.", | |
| "fr-FR-DeniseNeural-Female", | |
| ], | |
| [ | |
| "Dies ist eine Demo zur Umwandlung von Deutsch in Sprache.", | |
| "de-DE-AmalaNeural-Female", | |
| ], | |
| [ | |
| "Tämä on suomenkielinen tekstistä puheeksi -esittely.", | |
| "fi-FI-NooraNeural-Female", | |
| ], | |
| [ | |
| "Это демонстрационный пример преобразования русского текста в речь.", | |
| "ru-RU-SvetlanaNeural-Female", | |
| ], | |
| [ | |
| "Αυτή είναι μια επίδειξη μετατροπής ελληνικού κειμένου σε ομιλία.", | |
| "el-GR-AthinaNeural-Female", | |
| ], | |
| [ | |
| "Esta es una demostración de conversión de texto a voz en español.", | |
| "es-ES-ElviraNeural-Female", | |
| ], | |
| [ | |
| "Questa è una dimostrazione di sintesi vocale in italiano.", | |
| "it-IT-ElsaNeural-Female", | |
| ], | |
| [ | |
| "Esta é uma demonstração de conversão de texto em fala em português.", | |
| "pt-PT-RaquelNeural-Female", | |
| ], | |
| [ | |
| "Це демонстрація тексту до мовлення українською мовою.", | |
| "uk-UA-PolinaNeural-Female", | |
| ], | |
| [ | |
| "هذا عرض توضيحي عربي لتحويل النص إلى كلام.", | |
| "ar-EG-SalmaNeural-Female", | |
| ], | |
| [ | |
| "இது தமிழ் உரையிலிருந்து பேச்சு மாற்ற டெமோ.", | |
| "ta-IN-PallaviNeural-Female", | |
| ], | |
| ], | |
| inputs=[tts_text, tts_voice], | |
| ) | |
| app.launch() | |