feat: add app.py

DEMO.md

@@ -0,0 +1,17 @@
+# 原語會族語語音合成系統
+
+ILRDF Formosan Text-To-Speech System
+
+## 研發團隊
+
+- [李鴻欣 Hung-Shin Lee](mailto:hungshinlee@gmail.com)
+- [陳力瑋 Li-Wei Chen](mailto:wayne900619@gmail.com)
+- [意傳科技](https://ithuan.tw/)
+- [原住民族語言研究發展基金會](https://www.ilrdf.org.tw/)
+
+## 特別致謝
+- [聯和科創](https://www.104.com.tw/company/1a2x6bmu75)
+- [台灣阿美族語言永續發展學會/原民會阿美族語言推動組織](https://www.facebook.com/groups/ypspt/about)
+- [台灣太魯閣族語言發展學會](https://qkktt.com/)
+- [台灣原住民族賽德克族語言文化學會](https://www.facebook.com/3S3TBL/)
+- 族語老師們

app.py

@@ -1,7 +1,257 @@
 import gradio as gr
+import numpy as np
 
-def greet(name):
-    return "Hello " + name + "!!"
+from ipa import g2p
+from ipa.ipa import text_to_ipa
+from models import models_config
+
+
+def _do_tts(model_id, ipa, language_name, speaker_name=None, speaker_wav=None):
+    model = models_config[model_id]["model"]
+    if speaker_wav is not None:
+        return model.tts(
+            ipa,
+            speaker_wav=speaker_wav,
+            language_name=language_name,
+            split_sentences=False,
+        )
+    return model.tts(
+        ipa,
+        speaker_name=speaker_name,
+        language_name=language_name,
+        split_sentences=False,
+    )
+
+
+def text_to_speech(
+    model_id: str,
+    use_default_emb_or_custom: str,
+    speaker_wav,
+    speaker: str,
+    language: str,
+    dialect: str,
+    speed: float,
+    text: str,
+):
+    if len(text) == 0:
+        raise gr.Error("請勿輸入空字串。")
+    tag = language
+    if language not in g2p:
+        tag = f"{language}_{dialect}"
+
+    ipa = text_to_ipa(text, tag, g2p)
+
+    models_config[model_id]["model"].tts_model.length_scale = speed
+    if use_default_emb_or_custom == "預設語者":
+        wav = _do_tts(
+            model_id,
+            ipa,
+            speaker_name=speaker
+            if len(models_config[model_id]["speaker_mapping"]) > 1
+            else None,
+            language_name=language,
+        )
+    else:
+        wav = _do_tts(
+            model_id,
+            ipa,
+            speaker_wav=speaker_wav,
+            language_name=language,
+        )
+
+    return (
+        models_config[model_id]["model"].tts_model.config.audio.sample_rate,
+        np.array(wav),
+    )
+
+
+def when_model_selected(model_id):
+    model_config = models_config[model_id]
+
+    speaker_drop_down_choices = [
+        (k, v) for k, v in model_config["speaker_mapping"].items()
+    ]
+
+    language_radio_choices = [
+        (k, v) for k, v in model_config["language_mapping"].items()
+    ]
+
+    use_default_emb_or_ref_radio_visible = False
+    if model_config["model"].tts_model.config.model_args.speaker_encoder_model_path:
+        use_default_emb_or_ref_radio_visible = True
+
+    return (
+        gr.update(
+            choices=speaker_drop_down_choices,
+            value=speaker_drop_down_choices[0][1]
+            if len(speaker_drop_down_choices) > 0
+            else None,
+            interactive=len(speaker_drop_down_choices) > 1,
+        ),
+        gr.update(
+            choices=language_radio_choices,
+            value=language_radio_choices[0][1],
+            interactive=len(language_radio_choices) > 1,
+        ),
+        gr.update(visible=use_default_emb_or_ref_radio_visible, value="預設語者"),
+    )
+
+
+def use_default_emb_or_custom_radio_input(use_default_emb_or_custom):
+    if use_default_emb_or_custom == "客製化語者":
+        return gr.update(visible=True), gr.update(visible=False)
+    return gr.update(visible=False), gr.update(visible=True)
+
+
+def language_radio_changed(language):
+    if language in g2p:
+        return gr.update(visible=False)
+    dialect_choices = [tag.split("_")[1] for tag in g2p.keys() if language in tag]
+    return gr.update(
+        choices=dialect_choices,
+        value=dialect_choices[0],
+        interactive=len(dialect_choices) > 1,
+    )
+
+
+demo = gr.Blocks(
+    title="臺灣南島語語音合成系統",
+    css="@import url(https://tauhu.tw/tauhu-oo.css);",
+    theme=gr.themes.Default(
+        font=(
+            "tauhu-oo",
+            gr.themes.GoogleFont("Source Sans Pro"),
+            "ui-sans-serif",
+            "system-ui",
+            "sans-serif",
+        )
+    ),
+)
+
+with demo:
+    default_model_id = list(models_config.keys())[0]
+    model_drop_down = gr.Dropdown(
+        models_config.keys(),
+        value=default_model_id,
+        label="模型",
+    )
+    use_default_emb_or_custom_radio = gr.Radio(
+        label="語者類型",
+        choices=["預設語者", "客製化語者"],
+        value="客製化語者",
+        visible=True,
+        show_label=False,
+    )
+    speaker_wav = gr.Audio(
+        label="客製化語音",
+        visible=True,
+        editable=False,
+        type="filepath",
+        waveform_options=gr.WaveformOptions(
+            show_controls=False,
+            sample_rate=16000,
+        ),
+    )
+    speaker_drop_down = gr.Dropdown(
+        choices=[
+            (k, v)
+            for k, v in models_config[default_model_id]["speaker_mapping"].items()
+        ],
+        value=list(models_config[default_model_id]["speaker_mapping"].values())[0],
+        label="語者",
+        interactive=len(models_config[default_model_id]["speaker_mapping"]) > 1,
+        visible=False,
+    )
+    use_default_emb_or_custom_radio.change(
+        use_default_emb_or_custom_radio_input,
+        inputs=[use_default_emb_or_custom_radio],
+        outputs=[speaker_wav, speaker_drop_down],
+    )
+
+    default_language = list(
+        models_config[default_model_id]["language_mapping"].values()
+    )[0]
+    language_radio = gr.Radio(
+        choices=[
+            (k, v)
+            for k, v in models_config[default_model_id]["language_mapping"].items()
+        ],
+        value=default_language,
+        label="語言",
+        interactive=len(models_config[default_model_id]["language_mapping"]) > 1,
+    )
+
+    default_dialect_choices = [
+        tag.split("_")[1] for tag in g2p.keys() if default_language in tag
+    ]
+    dialect_radio = gr.Radio(
+        choices=default_dialect_choices,
+        value=default_dialect_choices[0],
+        label="方言",
+        interactive=len(default_dialect_choices) > 1,
+    )
+
+    language_radio.change(
+        language_radio_changed, inputs=[language_radio], outputs=[dialect_radio]
+    )
+
+    model_drop_down.input(
+        when_model_selected,
+        inputs=[model_drop_down],
+        outputs=[speaker_drop_down, language_radio, use_default_emb_or_custom_radio],
+    )
+
+    input_text = gr.Textbox(
+        label="輸入文字",
+        value="",
+    )
+
+    speed = gr.Slider(maximum=1.5, minimum=0.5, value=1, label="語速")
+
+    with open("DEMO.md") as tong:
+        gr.Markdown(tong.read())
+    gr.Interface(
+        text_to_speech,
+        inputs=[
+            model_drop_down,
+            use_default_emb_or_custom_radio,
+            speaker_wav,
+            speaker_drop_down,
+            language_radio,
+            dialect_radio,
+            speed,
+            input_text,
+        ],
+        outputs=[
+            gr.Audio(interactive=False, label="合成語音", show_download_button=True),
+        ],
+        allow_flagging="auto",
+    )
+    gr.Examples(
+        [
+            [
+                "預設語者",
+                "formosan_dict_ami#wav/formosan_dict_ami/000002_2.31-6.09.wav",
+                "阿美",
+                "南勢",
+                "mikadavu ku vavainay, i vavahiyan, a luma’",
+            ],
+            [
+                "預設語者",
+                "formosan_dict_ami#wav/formosan_dict_ami/000035_0.00-3.69.wav",
+                "阿美",
+                "南勢",
+                "mikadavu ku vavainay, i vavahiyan, a luma’",
+            ],
+        ],
+        label="範例",
+        inputs=[
+            use_default_emb_or_custom_radio,
+            speaker_drop_down,
+            language_radio,
+            dialect_radio,
+            input_text,
+        ],
+    )
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
 demo.launch()

configs/g2p.yaml

@@ -0,0 +1 @@
+g2p: ${load_g2p:${gh_download:FormoSpeech/FormoLexicon, formosan/g2p.csv, ${oc.env:GH_TOKEN}}}

configs/models.yaml

@@ -0,0 +1,7 @@
+yourtts-ami:
+  model: ${load_model:united-link/yourtts-formosan-ami}
+  language_mapping:
+    阿美: 阿美
+  speaker_mapping: # display_name: id
+    男: formosan_dict_ami#wav/formosan_dict_ami/000002_2.31-6.09.wav
+    女: formosan_dict_ami#wav/formosan_dict_ami/000035_0.00-3.69.wav

ipa/__init__.py

@@ -0,0 +1,76 @@
+import csv
+from io import BytesIO
+
+import requests
+from omegaconf import OmegaConf
+
+EXTRA_G2P = {
+    "z": "z",
+    "o": "o",
+    "h": "h",
+    "g": "g",
+    "y": "j",
+    "w": "w",
+    "c": "ʦ",
+    "u": "u",
+    "f": "f",
+    "v": "v",
+    "j": "ɟ",
+    "b": "b",
+    "q": "q",
+    "e": "e",
+    ",": ",",
+}
+
+
+def gh_download(repo, path, token):
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/vnd.github.raw+json",
+    }
+
+    url = f"https://api.github.com/repos/{repo}/contents/{path}"
+    response = requests.get(url, headers=headers)
+    if response.status_code != 200:
+        raise Exception(f"Failed to download {path} from {repo}, response: {response}")
+    response.encoding = "utf-8-sig"
+
+    return response.text
+
+
+def load_g2p(g2p_string):
+    g2p = dict()
+
+    csv_reader = csv.DictReader(g2p_string.split("\n"))
+
+    for row in csv_reader:
+        # print(row)
+        language = row["Language"]
+        dialect = row["Dialect"]
+
+        if dialect == "-":
+            lang_tag = f"{language}"
+        else:
+            lang_tag = f"{language}_{dialect}"
+
+        for key in row:
+            if key in ["Language", "Dialect"]:
+                continue
+
+            if row[key] == "-":
+                continue
+
+            g2p[lang_tag] = g2p.get(lang_tag, {})
+            g2p[lang_tag][key] = row[key].split(",")[0]
+
+        for g, p in EXTRA_G2P.items():
+            if g not in g2p[lang_tag]:
+                g2p[lang_tag][g] = p
+
+    return g2p
+
+
+OmegaConf.register_new_resolver("gh_download", gh_download)
+OmegaConf.register_new_resolver("load_g2p", load_g2p)
+
+g2p = OmegaConf.to_object(OmegaConf.load("configs/g2p.yaml"))["g2p"]

ipa/ipa.py

@@ -0,0 +1,37 @@
+def can_form_string(x, symbol_dict):
+    def helper(x, symbol_dict, matched_parts):
+        if not x:
+            return True, matched_parts
+
+        for key in symbol_dict.keys():
+            if x.startswith(key):
+                result, parts = helper(
+                    x[len(key) :], symbol_dict, matched_parts + [key]
+                )
+                if result:
+                    return True, parts
+
+        return False, []
+
+    return helper(x, symbol_dict, [])
+
+
+def text_to_ipa(text, lang_tag, g2p):
+    ipa = []
+    words = text.split()
+
+    print(words)
+
+    for word in words:
+        ipa_parts = ""
+        result, matched_parts = can_form_string(word, g2p[lang_tag])
+
+        if result is False:
+            return ""
+
+        for matched_part in matched_parts:
+            ipa_parts = ipa_parts + g2p[lang_tag][matched_part]
+
+        ipa.append(ipa_parts)
+    ipa = " ".join(ipa)
+    return ipa

models/__init__.py

@@ -0,0 +1,37 @@
+import os
+
+import torch
+from huggingface_hub import snapshot_download
+from omegaconf import OmegaConf
+from TTS.utils.synthesizer import Synthesizer
+
+
+def load_model(model_id):
+    model_dir = snapshot_download(model_id)
+    config_file_path = os.path.join(model_dir, "config.json")
+    model_ckpt_path = os.path.join(model_dir, "model.pth")
+    speaker_file_path = os.path.join(model_dir, "speakers.pth")
+    language_file_path = os.path.join(model_dir, "language_ids.json")
+    speaker_embedding_file_path = os.path.join(model_dir, "speaker_embs.pth")
+
+    temp_config_path = "temp_config.json"
+    with open(config_file_path, "r") as f:
+        content = f.read()
+        content = content.replace("speakers.pth", speaker_file_path)
+        content = content.replace("language_ids.json", language_file_path)
+        content = content.replace("speaker_embs.pth", speaker_embedding_file_path)
+        f.close()
+    with open(temp_config_path, "w") as f:
+        f.write(content)
+        f.close()
+
+    return Synthesizer(
+        tts_checkpoint=model_ckpt_path,
+        tts_config_path=temp_config_path,
+        use_cuda=torch.cuda.is_available(),
+    )
+
+
+OmegaConf.register_new_resolver("load_model", load_model)
+
+models_config = OmegaConf.to_object(OmegaConf.load("configs/models.yaml"))

requirements.txt

@@ -0,0 +1,2 @@
+TTS
+omegaconf