update

modules/repos_static/resemble_enhance/inference.py

@@ -127,6 +127,8 @@ def inference(
 ):
     if config.runtime_env_vars.off_tqdm:
         trange = range
+    else:
+        from tqdm import trange
 
     remove_weight_norm_recursively(model)
 

modules/utils/audio.py

@@ -19,7 +19,11 @@ def audio_to_int16(audio_data):
     return audio_data
 
 
-def audiosegment_to_librosawav(audiosegment):
+def audiosegment_to_librosawav(audiosegment: AudioSegment) -> np.ndarray:
+    """
+    Converts pydub audio segment into np.float32 of shape [duration_in_seconds*sample_rate, channels],
+    where each value is in range [-1.0, 1.0].
+    """
     channel_sounds = audiosegment.split_to_mono()
     samples = [s.get_array_of_samples() for s in channel_sounds]
 

modules/webui/app.py

@@ -8,10 +8,11 @@ from modules import config
 from modules.webui import webui_config
 
 from modules.webui.changelog_tab import create_changelog_tab
+from modules.webui.ssml.podcast_tab import create_ssml_podcast_tab
 from modules.webui.system_tab import create_system_tab
 from modules.webui.tts_tab import create_tts_interface
-from modules.webui.ssml_tab import create_ssml_interface
-from modules.webui.spliter_tab import create_spliter_tab
+from modules.webui.ssml.ssml_tab import create_ssml_interface
+from modules.webui.ssml.spliter_tab import create_spliter_tab
 from modules.webui.speaker_tab import create_speaker_panel
 from modules.webui.readme_tab import create_readme_tab
 
@@ -86,10 +87,17 @@ def create_interface():
                 create_tts_interface()
 
             with gr.TabItem("SSML", id="ssml"):
-                ssml_input = create_ssml_interface()
-
-            with gr.TabItem("Spilter"):
-                create_spliter_tab(ssml_input, tabs=tabs)
+                with gr.Tabs() as ssml_tabs:
+                    with gr.TabItem("Editor", id="ssml.editor"):
+                        ssml_input = create_ssml_interface()
+                    with gr.TabItem("Spilter"):
+                        create_spliter_tab(
+                            ssml_input=ssml_input, tabs1=tabs, tabs2=ssml_tabs
+                        )
+                    with gr.TabItem("Podcast"):
+                        create_ssml_podcast_tab(
+                            ssml_input=ssml_input, tabs1=tabs, tabs2=ssml_tabs
+                        )
 
             with gr.TabItem("Speaker"):
                 create_speaker_panel()

modules/webui/speaker/speaker_merger.py

@@ -3,6 +3,7 @@ import gradio as gr
 import torch
 
 from modules.hf import spaces
+from modules.webui import webui_utils
 from modules.webui.webui_utils import get_speakers, tts_generate
 from modules.speaker import speaker_mgr, Speaker
 
@@ -138,23 +139,19 @@ merge_desc = """
 """
 
 
-def get_spk_choices():
-    speakers = get_speakers()
-
-    speaker_names = ["None"] + [get_speaker_show_name(speaker) for speaker in speakers]
-    return speaker_names
-
-
 # 显示 a b c d 四个选择框，选择一个或多个，然后可以试音，并导出
 def create_speaker_merger():
-    speaker_names = get_spk_choices()
+    def get_spk_choices():
+        speakers, speaker_names = webui_utils.get_speaker_names()
+        speaker_names = ["None"] + speaker_names
+        return speaker_names
 
     gr.Markdown(merge_desc)
 
     def spk_picker(label_tail: str):
         with gr.Row():
             spk_a = gr.Dropdown(
-                choices=speaker_names, value="None", label=f"Speaker {label_tail}"
+                choices=get_spk_choices(), value="None", label=f"Speaker {label_tail}"
             )
             refresh_a_btn = gr.Button("🔄", variant="secondary")
 

modules/webui/ssml/podcast_tab.py

@@ -0,0 +1,210 @@
+import gradio as gr
+import pandas as pd
+import torch
+
+from modules.normalization import text_normalize
+from modules.webui import webui_utils
+from modules.hf import spaces
+
+podcast_default_case = [
+    [1, "female2", "你好，欢迎收听今天的播客内容。今天我们要聊的是中华料理。", "chat"],
+    [2, "Alice", "嗨，我特别期待这个话题！中华料理真的是博大精深。", "chat"],
+    [
+        3,
+        "Bob",
+        "没错，中华料理有着几千年的历史，而且每个地区都有自己的特色菜。",
+        "chat",
+    ],
+    [
+        4,
+        "female2",
+        "那我们先从最有名的川菜开始吧。川菜以其麻辣著称，是很多人的最爱。",
+        "chat",
+    ],
+    [
+        5,
+        "Alice",
+        "对，我特别喜欢吃麻婆豆腐和辣子鸡。那种麻辣的感觉真是让人难以忘怀。",
+        "chat",
+    ],
+    [
+        6,
+        "Bob",
+        "除了川菜，粤菜也是很受欢迎的。粤菜讲究鲜美，像是白切鸡和蒸鱼都是经典。",
+        "chat",
+    ],
+    [7, "female2", "对啊，粤菜的烹饪方式比较清淡，更注重食材本身的味道。", "chat"],
+    [8, "Alice", "还有北京的京菜，像北京烤鸭，那可是来北京必吃的美食。", "chat"],
+    [
+        9,
+        "Bob",
+        "不仅如此，还有淮扬菜、湘菜、鲁菜等等，每个菜系都有其独特的风味。",
+        "chat",
+    ],
+    [
+        10,
+        "female2",
+        "对对对，像淮扬菜的狮子头，湘菜的剁椒鱼头，都是让人垂涎三尺的美味。",
+        "chat",
+    ],
+]
+
+
+# NOTE: 因为 text_normalize 需要使用 tokenizer
+@torch.inference_mode()
+@spaces.GPU
+def merge_dataframe_to_ssml(msg, spk, style, df: pd.DataFrame):
+    ssml = ""
+    indent = " " * 2
+
+    for i, row in df.iterrows():
+        text = row.get("text")
+        spk = row.get("speaker")
+        style = row.get("style")
+
+        ssml += f"{indent}<voice"
+        if spk:
+            ssml += f' spk="{spk}"'
+        if style:
+            ssml += f' style="{style}"'
+        ssml += ">\n"
+        ssml += f"{indent}{indent}{text_normalize(text)}\n"
+        ssml += f"{indent}</voice>\n"
+    # 原封不动输出回去是为了触发 loadding 效果
+    return msg, spk, style, f"<speak version='0.1'>\n{ssml}</speak>"
+
+
+def create_ssml_podcast_tab(ssml_input: gr.Textbox, tabs1: gr.Tabs, tabs2: gr.Tabs):
+    def get_spk_choices():
+        speakers, speaker_names = webui_utils.get_speaker_names()
+        speaker_names = ["-1"] + speaker_names
+        return speaker_names
+
+    styles = ["*auto"] + [s.get("name") for s in webui_utils.get_styles()]
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            with gr.Group():
+                spk_input_dropdown = gr.Dropdown(
+                    choices=get_spk_choices(),
+                    interactive=True,
+                    value="female : female2",
+                    show_label=False,
+                )
+                style_input_dropdown = gr.Dropdown(
+                    choices=styles,
+                    # label="Choose Style",
+                    interactive=True,
+                    show_label=False,
+                    value="*auto",
+                )
+            with gr.Group():
+                msg = gr.Textbox(
+                    lines=5, label="Message", placeholder="Type speaker message here"
+                )
+                add = gr.Button("Add")
+                undo = gr.Button("Undo")
+                clear = gr.Button("Clear")
+        with gr.Column(scale=5):
+            with gr.Group():
+                gr.Markdown("📔Script")
+                script_table = gr.DataFrame(
+                    headers=["index", "speaker", "text", "style"],
+                    datatype=["number", "str", "str", "str"],
+                    interactive=False,
+                    wrap=True,
+                    value=podcast_default_case,
+                    row_count=(0, "dynamic"),
+                )
+
+    send_to_ssml_btn = gr.Button("📩Send to SSML", variant="primary")
+
+    def add_message(msg, spk, style, sheet: pd.DataFrame):
+        if not msg:
+            return "", sheet
+
+        data = pd.DataFrame(
+            {
+                "index": [sheet.shape[0]],
+                "speaker": [spk.split(" : ")[1].strip()],
+                "text": [msg],
+                "style": [style],
+            },
+        )
+
+        # 如果只有一行 并且是空的
+        is_empty = sheet.empty or (sheet.shape[0] == 1 and "text" not in sheet.iloc[0])
+
+        if is_empty:
+            sheet = data
+        else:
+            sheet = pd.concat(
+                [
+                    sheet,
+                    data,
+                ],
+                ignore_index=True,
+            )
+        return "", sheet
+
+    def undo_message(msg, spk, style, sheet: pd.DataFrame):
+        if sheet.empty:
+            return msg, spk, style, sheet
+        data = sheet.iloc[-1]
+        sheet = sheet.iloc[:-1]
+        spk = ""
+        for choice in get_spk_choices():
+            if choice.endswith(data["speaker"]) and " : " in choice:
+                spk = choice
+                break
+        return data["text"], spk, data["style"], sheet
+
+    def clear_message():
+        return "", pd.DataFrame(
+            columns=["index", "speaker", "text", "style"],
+        )
+
+    def send_to_ssml(msg, spk, style, sheet: pd.DataFrame):
+        if sheet.empty:
+            return gr.Error("Please add some text to the script table.")
+        msg, spk, style, ssml = merge_dataframe_to_ssml(msg, spk, style, sheet)
+        return [
+            msg,
+            spk,
+            style,
+            gr.Textbox(value=ssml),
+            gr.Tabs(selected="ssml"),
+            gr.Tabs(selected="ssml.editor"),
+        ]
+
+    msg.submit(
+        add_message,
+        inputs=[msg, spk_input_dropdown, style_input_dropdown, script_table],
+        outputs=[msg, script_table],
+    )
+    add.click(
+        add_message,
+        inputs=[msg, spk_input_dropdown, style_input_dropdown, script_table],
+        outputs=[msg, script_table],
+    )
+    undo.click(
+        undo_message,
+        inputs=[msg, spk_input_dropdown, style_input_dropdown, script_table],
+        outputs=[msg, spk_input_dropdown, style_input_dropdown, script_table],
+    )
+    clear.click(
+        clear_message,
+        outputs=[msg, script_table],
+    )
+    send_to_ssml_btn.click(
+        send_to_ssml,
+        inputs=[msg, spk_input_dropdown, style_input_dropdown, script_table],
+        outputs=[
+            msg,
+            spk_input_dropdown,
+            style_input_dropdown,
+            ssml_input,
+            tabs1,
+            tabs2,
+        ],
+    )

modules/webui/ssml/spliter_tab.py

@@ -0,0 +1,169 @@
+import gradio as gr
+import torch
+from modules.normalization import text_normalize
+from modules.webui import webui_utils
+from modules.webui.webui_utils import (
+    get_speakers,
+    get_styles,
+    split_long_text,
+)
+from modules.hf import spaces
+
+
+# NOTE: 因为 text_normalize 需要使用 tokenizer
+@torch.inference_mode()
+@spaces.GPU
+def merge_dataframe_to_ssml(dataframe, spk, style, seed):
+    if style == "*auto":
+        style = None
+    if spk == "-1" or spk == -1:
+        spk = None
+    if seed == -1 or seed == "-1":
+        seed = None
+
+    ssml = ""
+    indent = " " * 2
+
+    for i, row in dataframe.iterrows():
+        ssml += f"{indent}<voice"
+        if spk:
+            ssml += f' spk="{spk}"'
+        if style:
+            ssml += f' style="{style}"'
+        if seed:
+            ssml += f' seed="{seed}"'
+        ssml += ">\n"
+        ssml += f"{indent}{indent}{text_normalize(row.iloc[1])}\n"
+        ssml += f"{indent}</voice>\n"
+    # 原封不动输出回去是为了触发 loadding 效果
+    return dataframe, spk, style, seed, f"<speak version='0.1'>\n{ssml}</speak>"
+
+
+# 长文本处理
+# 可以输入长文本，并选择切割方法，切割之后可以将拼接的SSML发送到SSML tab
+# 根据 。 句号切割，切割之后显示到 data table
+def create_spliter_tab(ssml_input, tabs1, tabs2):
+    speakers, speaker_names = webui_utils.get_speaker_names()
+    speaker_names = ["*random"] + speaker_names
+
+    styles = ["*auto"] + [s.get("name") for s in get_styles()]
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            # 选择说话人 选择风格 选择seed
+            with gr.Group():
+                gr.Markdown("🗣️Speaker")
+                spk_input_text = gr.Textbox(
+                    label="Speaker (Text or Seed)",
+                    value="female2",
+                    show_label=False,
+                )
+                spk_input_dropdown = gr.Dropdown(
+                    choices=speaker_names,
+                    interactive=True,
+                    value="female : female2",
+                    show_label=False,
+                )
+                spk_rand_button = gr.Button(
+                    value="🎲",
+                    variant="secondary",
+                )
+            with gr.Group():
+                gr.Markdown("🎭Style")
+                style_input_dropdown = gr.Dropdown(
+                    choices=styles,
+                    interactive=True,
+                    show_label=False,
+                    value="*auto",
+                )
+            with gr.Group():
+                gr.Markdown("🗣️Seed")
+                infer_seed_input = gr.Number(
+                    value=42,
+                    label="Inference Seed",
+                    show_label=False,
+                    minimum=-1,
+                    maximum=2**32 - 1,
+                )
+                infer_seed_rand_button = gr.Button(
+                    value="🎲",
+                    variant="secondary",
+                )
+
+            send_btn = gr.Button("📩Send to SSML", variant="primary")
+
+        with gr.Column(scale=3):
+            with gr.Group():
+                gr.Markdown("📝Long Text Input")
+                gr.Markdown("- 此页面用于处理超长文本")
+                gr.Markdown("- 切割后，可以选择说话人、风格、seed，然后发送到SSML")
+                long_text_input = gr.Textbox(
+                    label="Long Text Input",
+                    lines=10,
+                    placeholder="输入长文本",
+                    elem_id="long-text-input",
+                    show_label=False,
+                )
+                long_text_split_button = gr.Button("🔪Split Text")
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            with gr.Group():
+                gr.Markdown("🎨Output")
+                long_text_output = gr.DataFrame(
+                    headers=["index", "text", "length"],
+                    datatype=["number", "str", "number"],
+                    elem_id="long-text-output",
+                    interactive=False,
+                    wrap=True,
+                    value=[],
+                )
+
+    spk_input_dropdown.change(
+        fn=lambda x: x.startswith("*") and "-1" or x.split(":")[-1].strip(),
+        inputs=[spk_input_dropdown],
+        outputs=[spk_input_text],
+    )
+    spk_rand_button.click(
+        lambda x: int(torch.randint(0, 2**32 - 1, (1,)).item()),
+        inputs=[spk_input_text],
+        outputs=[spk_input_text],
+    )
+    infer_seed_rand_button.click(
+        lambda x: int(torch.randint(0, 2**32 - 1, (1,)).item()),
+        inputs=[infer_seed_input],
+        outputs=[infer_seed_input],
+    )
+    long_text_split_button.click(
+        split_long_text,
+        inputs=[long_text_input],
+        outputs=[long_text_output],
+    )
+
+    infer_seed_rand_button.click(
+        lambda x: int(torch.randint(0, 2**32 - 1, (1,)).item()),
+        inputs=[infer_seed_input],
+        outputs=[infer_seed_input],
+    )
+
+    send_btn.click(
+        merge_dataframe_to_ssml,
+        inputs=[
+            long_text_output,
+            spk_input_text,
+            style_input_dropdown,
+            infer_seed_input,
+        ],
+        outputs=[
+            long_text_output,
+            spk_input_text,
+            style_input_dropdown,
+            infer_seed_input,
+            ssml_input,
+        ],
+    )
+
+    def change_tab():
+        return gr.Tabs(selected="ssml"), gr.Tabs(selected="ssml.editor")
+
+    send_btn.click(change_tab, inputs=[], outputs=[tabs1, tabs2])

modules/webui/ssml/ssml_tab.py

@@ -0,0 +1,61 @@
+import gradio as gr
+from modules.webui.webui_utils import (
+    synthesize_ssml,
+)
+from modules.webui import webui_config
+from modules.webui.examples import ssml_examples, default_ssml
+
+
+def create_ssml_interface():
+    with gr.Row():
+        with gr.Column(scale=3):
+            with gr.Group():
+                gr.Markdown("📝SSML Input")
+                gr.Markdown(f"- 最长{webui_config.ssml_max:,}字符，超过会被截断")
+                gr.Markdown("- 尽量保证使用相同的 seed")
+                gr.Markdown(
+                    "- 关于SSML可以看这个 [文档](https://github.com/lenML/ChatTTS-Forge/blob/main/docs/SSML.md)"
+                )
+                ssml_input = gr.Textbox(
+                    label="SSML Input",
+                    lines=10,
+                    value=default_ssml,
+                    placeholder="输入 SSML 或选择示例",
+                    elem_id="ssml_input",
+                    show_label=False,
+                )
+                ssml_button = gr.Button("🔊Synthesize SSML", variant="primary")
+        with gr.Column(scale=1):
+            with gr.Group():
+                # 参数
+                gr.Markdown("🎛️Parameters")
+                # batch size
+                batch_size_input = gr.Slider(
+                    label="Batch Size",
+                    value=4,
+                    minimum=1,
+                    maximum=webui_config.max_batch_size,
+                    step=1,
+                )
+
+            with gr.Group():
+                gr.Markdown("💪🏼Enhance")
+                enable_enhance = gr.Checkbox(value=True, label="Enable Enhance")
+                enable_de_noise = gr.Checkbox(value=False, label="Enable De-noise")
+
+            with gr.Group():
+                gr.Markdown("🎄Examples")
+                gr.Examples(
+                    examples=ssml_examples,
+                    inputs=[ssml_input],
+                )
+
+    ssml_output = gr.Audio(label="Generated Audio", format="mp3")
+
+    ssml_button.click(
+        synthesize_ssml,
+        inputs=[ssml_input, batch_size_input, enable_enhance, enable_de_noise],
+        outputs=ssml_output,
+    )
+
+    return ssml_input

modules/webui/tts_tab.py

@@ -27,6 +27,7 @@ def create_tts_interface():
     speaker_names = ["*random"] + [
         get_speaker_show_name(speaker) for speaker in speakers
     ]
+    speaker_names.sort(key=lambda x: x.startswith("*") and "-1" or x)
 
     styles = ["*auto"] + [s.get("name") for s in get_styles()]
 
@@ -121,18 +122,10 @@ def create_tts_interface():
                     # tooltip="Random Seed",
                     variant="secondary",
                 )
+            # 感觉这个没必要设置...
             use_decoder_input = gr.Checkbox(
                 value=True, label="Use Decoder", visible=False
             )
-            with gr.Group():
-                gr.Markdown("🔧Prompt engineering")
-                prompt1_input = gr.Textbox(label="Prompt 1")
-                prompt2_input = gr.Textbox(label="Prompt 2")
-                prefix_input = gr.Textbox(label="Prefix")
-
-                prompt_audio = gr.File(
-                    label="prompt_audio", visible=webui_config.experimental
-                )
 
             infer_seed_rand_button.click(
                 lambda x: int(torch.randint(0, 2**32 - 1, (1,)).item()),
@@ -214,6 +207,16 @@ def create_tts_interface():
                 )
                 refine_button = gr.Button("✍️Refine Text")
 
+            with gr.Group():
+                gr.Markdown("🔧Prompt engineering")
+                prompt1_input = gr.Textbox(label="Prompt 1")
+                prompt2_input = gr.Textbox(label="Prompt 2")
+                prefix_input = gr.Textbox(label="Prefix")
+
+                prompt_audio = gr.File(
+                    label="prompt_audio", visible=webui_config.experimental
+                )
+
             with gr.Group():
                 gr.Markdown("🔊Generate")
                 disable_normalize_input = gr.Checkbox(

modules/webui/webui_utils.py

@@ -32,6 +32,20 @@ def get_speakers():
     return speaker_mgr.list_speakers()
 
 
+def get_speaker_names() -> tuple[list[Speaker], list[str]]:
+    speakers = get_speakers()
+
+    def get_speaker_show_name(spk):
+        if spk.gender == "*" or spk.gender == "":
+            return spk.name
+        return f"{spk.gender} : {spk.name}"
+
+    speaker_names = [get_speaker_show_name(speaker) for speaker in speakers]
+    speaker_names.sort(key=lambda x: x.startswith("*") and "-1" or x)
+
+    return speakers, speaker_names
+
+
 def get_styles():
     return styles_mgr.list_items()
 
@@ -93,7 +107,12 @@ def apply_audio_enhance(audio_data, sr, enable_denoise, enable_enhance):
 
 @torch.inference_mode()
 @spaces.GPU
-def synthesize_ssml(ssml: str, batch_size=4):
+def synthesize_ssml(
+    ssml: str,
+    batch_size=4,
+    enable_enhance=False,
+    enable_denoise=False,
+):
     try:
         batch_size = int(batch_size)
     except Exception:
@@ -116,7 +135,16 @@ def synthesize_ssml(ssml: str, batch_size=4):
     audio_segments = synthesize.synthesize_segments(segments)
     combined_audio = combine_audio_segments(audio_segments)
 
-    sr, audio_data = audio.pydub_to_np(combined_audio)
+    sr = combined_audio.frame_rate
+    audio_data, sr = apply_audio_enhance(
+        audio.audiosegment_to_librosawav(combined_audio),
+        sr,
+        enable_denoise,
+        enable_enhance,
+    )
+
+    # NOTE: 这里必须要加，不然 gradio 没法解析成 mp3 格式
+    audio_data = audio.audio_to_int16(audio_data)
 
     return sr, audio_data
 
@@ -193,6 +221,7 @@ def tts_generate(
     audio_data, sample_rate = apply_audio_enhance(
         audio_data, sample_rate, enable_denoise, enable_enhance
     )
+    # NOTE: 这里必须要加，不然 gradio 没法解析成 mp3 格式
     audio_data = audio.audio_to_int16(audio_data)
     return sample_rate, audio_data