Create app.py

app.py

@@ -0,0 +1,604 @@
+import sys
+from pathlib import Path
+
+repo_dir = Path("OpenVoice")
+
+if not repo_dir.exists():
+    !git clone https://github.com/myshell-ai/OpenVoice
+    orig_english_path = Path("OpenVoice/openvoice/text/_orig_english.py")
+    english_path = Path("OpenVoice/openvoice/text/english.py")
+
+    english_path.rename(orig_english_path)
+
+    with orig_english_path.open("r") as f:
+        data = f.read()
+        data = data.replace("unidecode", "anyascii")
+        with english_path.open("w") as out_f:
+            out_f.write(data)
+# append to sys.path so that modules from the repo could be imported
+sys.path.append(str(repo_dir))
+
+%pip install -q "librosa>=0.8.1" "wavmark>=0.0.3" "faster-whisper>=0.9.0" "pydub>=0.25.1" "whisper-timestamped>=1.14.2" "tqdm" "inflect>=7.0.0" "eng_to_ipa>=0.0.2" "pypinyin>=0.50.0" \
+"cn2an>=0.5.22" "jieba>=0.42.1" "langid>=1.1.6" "gradio>=4.15" "ipywebrtc" "anyascii" "openvino>=2023.3" "torch>=2.1" "nncf>=2.11.0"
+
+import os
+import torch
+import openvino as ov
+import ipywidgets as widgets
+from IPython.display import Audio
+
+core = ov.Core()
+
+from openvoice.api import BaseSpeakerTTS, ToneColorConverter, OpenVoiceBaseClass
+import openvoice.se_extractor as se_extractor
+
+CKPT_BASE_PATH = "checkpoints"
+
+en_suffix = f"{CKPT_BASE_PATH}/base_speakers/EN"
+zh_suffix = f"{CKPT_BASE_PATH}/base_speakers/ZH"
+converter_suffix = f"{CKPT_BASE_PATH}/converter"
+
+enable_chinese_lang = False
+
+def download_from_hf_hub(filename, local_dir="./"):
+    from huggingface_hub import hf_hub_download
+
+    os.makedirs(local_dir, exist_ok=True)
+    hf_hub_download(repo_id="myshell-ai/OpenVoice", filename=filename, local_dir=local_dir)
+
+
+download_from_hf_hub(f"{converter_suffix}/checkpoint.pth")
+download_from_hf_hub(f"{converter_suffix}/config.json")
+download_from_hf_hub(f"{en_suffix}/checkpoint.pth")
+download_from_hf_hub(f"{en_suffix}/config.json")
+
+download_from_hf_hub(f"{en_suffix}/en_default_se.pth")
+download_from_hf_hub(f"{en_suffix}/en_style_se.pth")
+
+if enable_chinese_lang:
+    download_from_hf_hub(f"{zh_suffix}/checkpoint.pth")
+    download_from_hf_hub(f"{zh_suffix}/config.json")
+    download_from_hf_hub(f"{zh_suffix}/zh_default_se.pth")
+
+pt_device = "cpu"
+
+en_base_speaker_tts = BaseSpeakerTTS(f"{en_suffix}/config.json", device=pt_device)
+en_base_speaker_tts.load_ckpt(f"{en_suffix}/checkpoint.pth")
+
+tone_color_converter = ToneColorConverter(f"{converter_suffix}/config.json", device=pt_device)
+tone_color_converter.load_ckpt(f"{converter_suffix}/checkpoint.pth")
+
+if enable_chinese_lang:
+    zh_base_speaker_tts = BaseSpeakerTTS(f"{zh_suffix}/config.json", device=pt_device)
+    zh_base_speaker_tts.load_ckpt(f"{zh_suffix}/checkpoint.pth")
+else:
+    zh_base_speaker_tts = None
+
+pt_device = "cpu"
+
+en_base_speaker_tts = BaseSpeakerTTS(f"{en_suffix}/config.json", device=pt_device)
+en_base_speaker_tts.load_ckpt(f"{en_suffix}/checkpoint.pth")
+
+tone_color_converter = ToneColorConverter(f"{converter_suffix}/config.json", device=pt_device)
+tone_color_converter.load_ckpt(f"{converter_suffix}/checkpoint.pth")
+
+if enable_chinese_lang:
+    zh_base_speaker_tts = BaseSpeakerTTS(f"{zh_suffix}/config.json", device=pt_device)
+    zh_base_speaker_tts.load_ckpt(f"{zh_suffix}/checkpoint.pth")
+else:
+    zh_base_speaker_tts = None
+
+class OVOpenVoiceBase(torch.nn.Module):
+    """
+    Base class for both TTS and voice tone conversion model: constructor is same for both of them.
+    """
+
+    def __init__(self, voice_model: OpenVoiceBaseClass):
+        super().__init__()
+        self.voice_model = voice_model
+        for par in voice_model.model.parameters():
+            par.requires_grad = False
+
+
+class OVOpenVoiceTTS(OVOpenVoiceBase):
+    """
+    Constructor of this class accepts BaseSpeakerTTS object for speech generation and wraps it's 'infer' method with forward.
+    """
+
+    def get_example_input(self):
+        stn_tst = self.voice_model.get_text("this is original text", self.voice_model.hps, False)
+        x_tst = stn_tst.unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([stn_tst.size(0)])
+        speaker_id = torch.LongTensor([1])
+        noise_scale = torch.tensor(0.667)
+        length_scale = torch.tensor(1.0)
+        noise_scale_w = torch.tensor(0.6)
+        return (
+            x_tst,
+            x_tst_lengths,
+            speaker_id,
+            noise_scale,
+            length_scale,
+            noise_scale_w,
+        )
+
+    def forward(self, x, x_lengths, sid, noise_scale, length_scale, noise_scale_w):
+        return self.voice_model.model.infer(x, x_lengths, sid, noise_scale, length_scale, noise_scale_w)
+
+
+class OVOpenVoiceConverter(OVOpenVoiceBase):
+    """
+    Constructor of this class accepts ToneColorConverter object for voice tone conversion and wraps it's 'voice_conversion' method with forward.
+    """
+
+    def get_example_input(self):
+        y = torch.randn([1, 513, 238], dtype=torch.float32)
+        y_lengths = torch.LongTensor([y.size(-1)])
+        target_se = torch.randn(*(1, 256, 1))
+        source_se = torch.randn(*(1, 256, 1))
+        tau = torch.tensor(0.3)
+        return (y, y_lengths, source_se, target_se, tau)
+
+    def forward(self, y, y_lengths, sid_src, sid_tgt, tau):
+        return self.voice_model.model.voice_conversion(y, y_lengths, sid_src, sid_tgt, tau)
+
+import nncf
+
+
+IRS_PATH = "openvino_irs/"
+EN_TTS_IR = f"{IRS_PATH}/openvoice_en_tts.xml"
+ZH_TTS_IR = f"{IRS_PATH}/openvoice_zh_tts.xml"
+VOICE_CONVERTER_IR = f"{IRS_PATH}/openvoice_tone_conversion.xml"
+
+paths = [EN_TTS_IR, VOICE_CONVERTER_IR]
+models = [
+    OVOpenVoiceTTS(en_base_speaker_tts),
+    OVOpenVoiceConverter(tone_color_converter),
+]
+if enable_chinese_lang:
+    models.append(OVOpenVoiceTTS(zh_base_speaker_tts))
+    paths.append(ZH_TTS_IR)
+ov_models = []
+
+for model, path in zip(models, paths):
+    if not os.path.exists(path):
+        ov_model = ov.convert_model(model, example_input=model.get_example_input())
+        ov_model = nncf.compress_weights(ov_model)
+        ov.save_model(ov_model, path)
+    else:
+        ov_model = core.read_model(path)
+    ov_models.append(ov_model)
+
+ov_en_tts, ov_voice_conversion = ov_models[:2]
+if enable_chinese_lang:
+    ov_zh_tts = ov_models[-1]
+
+REFERENCE_VOICES_PATH = f"{repo_dir}/resources/"
+reference_speakers = [
+    *[path for path in os.listdir(REFERENCE_VOICES_PATH) if os.path.splitext(path)[-1] == ".mp3"],
+    "record_manually",
+    "load_manually",
+]
+
+ref_speaker = widgets.Dropdown(
+    options=reference_speakers,
+    value=reference_speakers[0],
+    description="reference voice from which tone color will be copied",
+    disabled=False,
+)
+
+ref_speaker
+
+OUTPUT_DIR = "outputs/"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+ref_speaker_path = f"{REFERENCE_VOICES_PATH}/{ref_speaker.value}"
+allowed_audio_types = ".mp4,.mp3,.wav,.wma,.aac,.m4a,.m4b,.webm"
+
+if ref_speaker.value == "record_manually":
+    ref_speaker_path = f"{OUTPUT_DIR}/custom_example_sample.webm"
+    from ipywebrtc import AudioRecorder, CameraStream
+
+    camera = CameraStream(constraints={"audio": True, "video": False})
+    recorder = AudioRecorder(stream=camera, filename=ref_speaker_path, autosave=True)
+    display(recorder)
+elif ref_speaker.value == "load_manually":
+    upload_ref = widgets.FileUpload(
+        accept=allowed_audio_types,
+        multiple=False,
+        description="Select audio with reference voice",
+    )
+    display(upload_ref)
+
+def save_audio(voice_source: widgets.FileUpload, out_path: str):
+    with open(out_path, "wb") as output_file:
+        assert len(voice_source.value) > 0, "Please select audio file"
+        output_file.write(voice_source.value[0]["content"])
+
+
+if ref_speaker.value == "load_manually":
+    ref_speaker_path = f"{OUTPUT_DIR}/{upload_ref.value[0].name}"
+    save_audio(upload_ref, ref_speaker_path)
+
+en_source_default_se = torch.load(f"{en_suffix}/en_default_se.pth")
+en_source_style_se = torch.load(f"{en_suffix}/en_style_se.pth")
+zh_source_se = torch.load(f"{zh_suffix}/zh_default_se.pth") if enable_chinese_lang else None
+
+target_se, audio_name = se_extractor.get_se(ref_speaker_path, tone_color_converter, target_dir=OUTPUT_DIR, vad=True)
+
+def get_pathched_infer(ov_model: ov.Model, device: str) -> callable:
+    compiled_model = core.compile_model(ov_model, device)
+
+    def infer_impl(x, x_lengths, sid, noise_scale, length_scale, noise_scale_w):
+        ov_output = compiled_model((x, x_lengths, sid, noise_scale, length_scale, noise_scale_w))
+        return (torch.tensor(ov_output[0]),)
+
+    return infer_impl
+
+
+def get_patched_voice_conversion(ov_model: ov.Model, device: str) -> callable:
+    compiled_model = core.compile_model(ov_model, device)
+
+    def voice_conversion_impl(y, y_lengths, sid_src, sid_tgt, tau):
+        ov_output = compiled_model((y, y_lengths, sid_src, sid_tgt, tau))
+        return (torch.tensor(ov_output[0]),)
+
+    return voice_conversion_impl
+
+
+en_base_speaker_tts.model.infer = get_pathched_infer(ov_en_tts, device.value)
+tone_color_converter.model.voice_conversion = get_patched_voice_conversion(ov_voice_conversion, device.value)
+if enable_chinese_lang:
+    zh_base_speaker_tts.model.infer = get_pathched_infer(ov_zh_tts, device.value)
+
+voice_source = widgets.Dropdown(
+    options=["use TTS", "choose_manually"],
+    value="use TTS",
+    description="Voice source",
+    disabled=False,
+)
+
+voice_source
+
+if voice_source.value == "choose_manually":
+    upload_orig_voice = widgets.FileUpload(
+        accept=allowed_audio_types,
+        multiple=False,
+        description="audo whose tone will be replaced",
+    )
+    display(upload_orig_voice)
+
+if voice_source.value == "choose_manually":
+    orig_voice_path = f"{OUTPUT_DIR}/{upload_orig_voice.value[0].name}"
+    save_audio(upload_orig_voice, orig_voice_path)
+    source_se, _ = se_extractor.get_se(orig_voice_path, tone_color_converter, target_dir=OUTPUT_DIR, vad=True)
+else:
+    text = """
+    OpenVINO toolkit is a comprehensive toolkit for quickly developing applications and solutions that solve
+    a variety of tasks including emulation of human vision, automatic speech recognition, natural language processing,
+    recommendation systems, and many others.
+    """
+    source_se = en_source_default_se
+    orig_voice_path = f"{OUTPUT_DIR}/tmp.wav"
+    en_base_speaker_tts.tts(text, orig_voice_path, speaker="default", language="English")
+
+tau_slider = widgets.FloatSlider(
+    value=0.3,
+    min=0.01,
+    max=2.0,
+    step=0.01,
+    description="tau",
+    disabled=False,
+    readout_format=".2f",
+)
+tau_slider
+
+resulting_voice_path = f"{OUTPUT_DIR}/output_with_cloned_voice_tone.wav"
+
+tone_color_converter.convert(
+    audio_src_path=orig_voice_path,
+    src_se=source_se,
+    tgt_se=target_se,
+    output_path=resulting_voice_path,
+    tau=tau_slider.value,
+    message="@MyShell",
+)
+
+import gradio as gr
+import langid
+
+supported_languages = ["zh", "en"]
+
+
+def build_predict(
+    output_dir,
+    tone_color_converter,
+    en_tts_model,
+    zh_tts_model,
+    en_source_default_se,
+    en_source_style_se,
+    zh_source_se,
+):
+    def predict(prompt, style, audio_file_pth, agree):
+        return predict_impl(
+            prompt,
+            style,
+            audio_file_pth,
+            agree,
+            output_dir,
+            tone_color_converter,
+            en_tts_model,
+            zh_tts_model,
+            en_source_default_se,
+            en_source_style_se,
+            zh_source_se,
+        )
+
+    return predict
+
+
+def predict_impl(
+    prompt,
+    style,
+    audio_file_pth,
+    agree,
+    output_dir,
+    tone_color_converter,
+    en_tts_model,
+    zh_tts_model,
+    en_source_default_se,
+    en_source_style_se,
+    zh_source_se,
+):
+    text_hint = ""
+    if not agree:
+        text_hint += "[ERROR] Please accept the Terms & Condition!\n"
+        gr.Warning("Please accept the Terms & Condition!")
+        return (
+            text_hint,
+            None,
+            None,
+        )
+
+    language_predicted = langid.classify(prompt)[0].strip()
+    print(f"Detected language:{language_predicted}")
+
+    if language_predicted not in supported_languages:
+        text_hint += f"[ERROR] The detected language {language_predicted} for your input text is not in our Supported Languages: {supported_languages}\n"
+        gr.Warning(f"The detected language {language_predicted} for your input text is not in our Supported Languages: {supported_languages}")
+
+        return (
+            text_hint,
+            None,
+        )
+
+    if language_predicted == "zh":
+        tts_model = zh_tts_model
+        if zh_tts_model is None:
+            gr.Warning("TTS model for Chinece language was not loaded please set 'enable_chinese_lang=True`")
+            return (
+                text_hint,
+                None,
+            )
+        source_se = zh_source_se
+        language = "Chinese"
+        if style not in ["default"]:
+            text_hint += f"[ERROR] The style {style} is not supported for Chinese, which should be in ['default']\n"
+            gr.Warning(f"The style {style} is not supported for Chinese, which should be in ['default']")
+            return (
+                text_hint,
+                None,
+            )
+    else:
+        tts_model = en_tts_model
+        if style == "default":
+            source_se = en_source_default_se
+        else:
+            source_se = en_source_style_se
+        language = "English"
+        supported_styles = [
+            "default",
+            "whispering",
+            "shouting",
+            "excited",
+            "cheerful",
+            "terrified",
+            "angry",
+            "sad",
+            "friendly",
+        ]
+        if style not in supported_styles:
+            text_hint += f"[ERROR] The style {style} is not supported for English, which should be in {*supported_styles,}\n"
+            gr.Warning(f"The style {style} is not supported for English, which should be in {*supported_styles,}")
+            return (
+                text_hint,
+                None,
+            )
+
+    speaker_wav = audio_file_pth
+
+    if len(prompt) < 2:
+        text_hint += "[ERROR] Please give a longer prompt text \n"
+        gr.Warning("Please give a longer prompt text")
+        return (
+            text_hint,
+            None,
+        )
+    if len(prompt) > 200:
+        text_hint += (
+            "[ERROR] Text length limited to 200 characters for this demo, please try shorter text. You can clone our open-source repo and try for your usage \n"
+        )
+        gr.Warning("Text length limited to 200 characters for this demo, please try shorter text. You can clone our open-source repo for your usage")
+        return (
+            text_hint,
+            None,
+        )
+
+    # note diffusion_conditioning not used on hifigan (default mode), it will be empty but need to pass it to model.inference
+    try:
+        target_se, audio_name = se_extractor.get_se(speaker_wav, tone_color_converter, target_dir=OUTPUT_DIR, vad=True)
+    except Exception as e:
+        text_hint += f"[ERROR] Get target tone color error {str(e)} \n"
+        gr.Warning("[ERROR] Get target tone color error {str(e)} \n")
+        return (
+            text_hint,
+            None,
+        )
+
+    src_path = f"{output_dir}/tmp.wav"
+    tts_model.tts(prompt, src_path, speaker=style, language=language)
+
+    save_path = f"{output_dir}/output.wav"
+    encode_message = "@MyShell"
+    tone_color_converter.convert(
+        audio_src_path=src_path,
+        src_se=source_se,
+        tgt_se=target_se,
+        output_path=save_path,
+        message=encode_message,
+    )
+
+    text_hint += "Get response successfully \n"
+
+    return (
+        text_hint,
+        src_path,
+        save_path,
+    )
+
+
+description = """
+    # OpenVoice accelerated by OpenVINO:
+
+    a versatile instant voice cloning approach that requires only a short audio clip from the reference speaker to replicate their voice and generate speech in multiple languages. OpenVoice enables granular control over voice styles, including emotion, accent, rhythm, pauses, and intonation, in addition to replicating the tone color of the reference speaker. OpenVoice also achieves zero-shot cross-lingual voice cloning for languages not included in the massive-speaker training set.
+"""
+
+content = """
+<div>
+<strong>If the generated voice does not sound like the reference voice, please refer to <a href='https://github.com/myshell-ai/OpenVoice/blob/main/docs/QA.md'>this QnA</a>.</strong> <strong>For multi-lingual & cross-lingual examples, please refer to <a href='https://github.com/myshell-ai/OpenVoice/blob/main/demo_part2.ipynb'>this jupyter notebook</a>.</strong>
+This online demo mainly supports <strong>English</strong>. The <em>default</em> style also supports <strong>Chinese</strong>. But OpenVoice can adapt to any other language as long as a base speaker is provided.
+</div>
+"""
+wrapped_markdown_content = f"<div style='border: 1px solid #000; padding: 10px;'>{content}</div>"
+
+
+examples = [
+    [
+        "今天天气真好，我们一起出去吃饭吧。",
+        "default",
+        "OpenVoice/resources/demo_speaker1.mp3",
+        True,
+    ],
+    [
+        "This audio is generated by open voice with a half-performance model.",
+        "whispering",
+        "OpenVoice/resources/demo_speaker2.mp3",
+        True,
+    ],
+    [
+        "He hoped there would be stew for dinner, turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick, peppered, flour-fattened sauce.",
+        "sad",
+        "OpenVoice/resources/demo_speaker0.mp3",
+        True,
+    ],
+]
+
+
+def get_demo(
+    output_dir,
+    tone_color_converter,
+    en_tts_model,
+    zh_tts_model,
+    en_source_default_se,
+    en_source_style_se,
+    zh_source_se,
+):
+    with gr.Blocks(analytics_enabled=False) as demo:
+        with gr.Row():
+            gr.Markdown(description)
+        with gr.Row():
+            gr.HTML(wrapped_markdown_content)
+
+        with gr.Row():
+            with gr.Column():
+                input_text_gr = gr.Textbox(
+                    label="Text Prompt",
+                    info="One or two sentences at a time is better. Up to 200 text characters.",
+                    value="He hoped there would be stew for dinner, turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick, peppered, flour-fattened sauce.",
+                )
+                style_gr = gr.Dropdown(
+                    label="Style",
+                    info="Select a style of output audio for the synthesised speech. (Chinese only support 'default' now)",
+                    choices=[
+                        "default",
+                        "whispering",
+                        "cheerful",
+                        "terrified",
+                        "angry",
+                        "sad",
+                        "friendly",
+                    ],
+                    max_choices=1,
+                    value="default",
+                )
+                ref_gr = gr.Audio(
+                    label="Reference Audio",
+                    type="filepath",
+                    value="OpenVoice/resources/demo_speaker2.mp3",
+                )
+                tos_gr = gr.Checkbox(
+                    label="Agree",
+                    value=False,
+                    info="I agree to the terms of the cc-by-nc-4.0 license-: https://github.com/myshell-ai/OpenVoice/blob/main/LICENSE",
+                )
+
+                tts_button = gr.Button("Send", elem_id="send-btn", visible=True)
+
+            with gr.Column():
+                out_text_gr = gr.Text(label="Info")
+                audio_orig_gr = gr.Audio(label="Synthesised Audio", autoplay=False)
+                audio_gr = gr.Audio(label="Audio with cloned voice", autoplay=True)
+                # ref_audio_gr = gr.Audio(label="Reference Audio Used")
+                predict = build_predict(
+                    output_dir,
+                    tone_color_converter,
+                    en_tts_model,
+                    zh_tts_model,
+                    en_source_default_se,
+                    en_source_style_se,
+                    zh_source_se,
+                )
+
+                gr.Examples(
+                    examples,
+                    label="Examples",
+                    inputs=[input_text_gr, style_gr, ref_gr, tos_gr],
+                    outputs=[out_text_gr, audio_gr],
+                    fn=predict,
+                    cache_examples=False,
+                )
+                tts_button.click(
+                    predict,
+                    [input_text_gr, style_gr, ref_gr, tos_gr],
+                    outputs=[out_text_gr, audio_orig_gr, audio_gr],
+                )
+    return demo
+
+demo = get_demo(
+    OUTPUT_DIR,
+    tone_color_converter,
+    en_base_speaker_tts,
+    zh_base_speaker_tts,
+    en_source_default_se,
+    en_source_style_se,
+    zh_source_se,
+)
+demo.queue(max_size=2)
+
+try:
+    demo.launch(debug=True, height=1000)
+except Exception:
+    demo.launch(share=True, debug=True, height=1000)
+# if you are launching remotely, specify server_name and server_port
+# demo.launch(server_name='your server name', server_port='server port in int')
+# Read more in the docs: https://gradio.app/docs/