.pt speakers map

app.py

@@ -7,6 +7,7 @@ import gradio as gr
 from util import InitModels, load_config, Examples
 import numpy as np
 import torch
+import json
 
 config = load_config("./model_config.yaml")
 models_configs = config.models
@@ -18,9 +19,13 @@ examples = examples_maker()
 init_models = InitModels(models_configs)
 models = init_models()
 
+# Load speaker map
+with open("./speakers/speaker_map.json", "r") as f:
+    speaker_map = json.load(f)
+
 
 @spaces.GPU
-def generate_speech_gpu(text, model_choice, t, top_p, rp):
+def generate_speech_gpu(text, model_choice, speaker_choice, t, top_p, rp):
     """
     Generate speech from text using the selected model on GPU
     """
@@ -37,9 +42,13 @@ def generate_speech_gpu(text, model_choice, t, top_p, rp):
 
         selected_model = models[model_choice]
 
+        # Get speaker embedding path
+        speaker_emb = speaker_map.get(speaker_choice) if speaker_choice else None
+
         print(f"Generating speech with {model_choice}...")
         audio, _ = selected_model(
             text,
+            speaker_emb=speaker_emb,
             temperature=t,
             top_p=top_p,
             repetition_penalty=rp
@@ -67,6 +76,12 @@ with gr.Blocks(title="😻 KaniTTS - Text to Speech", theme=gr.themes.Ocean()) a
                 label="Selected Model"
             )
 
+            speaker_dropdown = gr.Dropdown(
+                choices=list(speaker_map.keys()),
+                value=list(speaker_map.keys())[0],
+                label="Speaker"
+            )
+
             text_input = gr.Textbox(
                 label="Text",
                 placeholder="Enter your text ...",
@@ -87,7 +102,7 @@ with gr.Blocks(title="😻 KaniTTS - Text to Speech", theme=gr.themes.Ocean()) a
                     minimum=1.0, maximum=2.0, value=1.1, step=0.05,
                     label="Repetition Penalty",
                 )
-            
+
             generate_btn = gr.Button("Run", variant="primary", size="lg")
 
             
@@ -100,17 +115,17 @@ with gr.Blocks(title="😻 KaniTTS - Text to Speech", theme=gr.themes.Ocean()) a
     # GPU generation event
     generate_btn.click(
         fn=generate_speech_gpu,
-        inputs=[text_input, model_dropdown, temp, top_p, rp],
+        inputs=[text_input, model_dropdown, speaker_dropdown, temp, top_p, rp],
         outputs=[audio_output]
     )
-    
+
     with gr.Row():
 
         examples = examples
 
         gr.Examples(
             examples=examples,
-            inputs=[text_input, model_dropdown, temp, top_p, rp],
+            inputs=[text_input, model_dropdown, speaker_dropdown, temp, top_p, rp],
             fn=generate_speech_gpu,
             outputs=[audio_output],
             cache_examples=True,

examples.yaml

@@ -1,21 +1,24 @@
 examples:
   - text: >-
       No, that does not make you a failure. No, sweetie, no. It just, uh, it just means that you're having a tough time...
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Kore (en)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1
 
   - text: >-
       Anyway, um, so, um, tell me, tell me all about her. I mean, what's she like? Is she really, you know, pretty?
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Andrew (en)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1
 
   - text: >-
       Have some wine, the March Hare said in an encouraging tone. Alice looked all round the table, but there was nothing on it but tea. I don't see any wine, she remarked. There isn't any, said the March Hare. Then it wasn't very civil of you to offer it, said Alice angrily. It wasn't very civil of you to sit down without being invited, said the March Hare. I didn't know it was YOUR table, said Alice; it's laid for a great many more than three. Your hair wants cutting, said the Hatter.  He had been looking at Alice for some time with great curiosity, and this was his first speech. You should learn not to make personal remarks, Alice said with some severity; it's very rude.
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Andrew (en)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1
@@ -23,21 +26,24 @@ examples:
 
   - text: >-
       Attention networks have proven to be an effective approach for embedding categorical inference within a deep neural network. However, for many tasks we may want to model richer structural dependencies without abandoning end-to-end training. In this work, we experiment with incorporating richer structural distributions, encoded using graphical models, within deep networks. We show that these structured attention networks are simple extensions of the basic attention procedure, and that they allow for extending attention beyond the standard softselection approach, such as attending to partial segmentations or to subtrees.
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Andrew (en)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1
 
   - text: >-
       Кыргыз жери! Сенин ар бир ташыңда, ар бир тооңдо, ар бир сууңда менин жүрөгүмдүн бир бөлүгү бар. Сен менин ата-журтум, менин ыйык мекенимсиң.
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Aisulu (ky)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1
 
   - text: >-
       Өлкө башчынын айтымында, айдоочулук күбөлүктөрдү алмаштыруу чечими коопсуздук жана мамлекеттин эл аралык аброю үчүн кабыл алынган. Анткени мурда чет өлкөлөрдө чыгарылган эски үлгүдөгү документтердин коргоо деңгээли алсыз болуп, жасалмалоо фактылары кеңири тараган. Айрым жасалма ID-паспорттор жана айдоочулук күбөлүктөр менен кылмышка, атүгүл террорчулукка байланышкан учурлар катталып, бул Кыргызстанга олуттуу имидждик зыян келтирген.
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Baike (ky)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1
@@ -45,7 +51,8 @@ examples:
 
   - text: >-
       ¡Qué alegría volver a verte después de tanto tiempo!
-    model: "test-135000"
+    model: "Exp-1"
+    speaker: "Nova (es)"
     temperature: 1
     top_p: 0.95
     repetition_penalty: 1.1

speakers/speaker_map.json

@@ -0,0 +1,9 @@
+{
+    "Kore (en)": "./speakers/speaker_1.pt",
+    "Puck (en)": "./speakers/speaker_2.pt",
+    "Andrew (en)": "./speakers/speaker_3.pt",
+    "Aisulu (ky)": "./speakers/speaker_4.pt",
+    "Baike (ky)": "./speakers/speaker_5.pt",
+    "Ash (es)": "./speakers/speaker_6.pt",
+    "Nova (es)": "./speakers/speaker_7.pt"
+}

util.py

@@ -1,4 +1,5 @@
 from kani_tts import KaniTTS
+from kani_tts import SpeakerEmbedder
 
 import os
 from omegaconf import OmegaConf
@@ -56,9 +57,6 @@ class InitModels:
             models[model_name] = KaniTTS(
                 model_name=cfg_dict.get('model_name'),
                 device_map=cfg_dict.get('device_map'),
-                use_bematts=cfg_dict.get('use_bematts', False),
-                audio_step=cfg_dict.get('audio_step', 1.0),
-                use_learnable_rope=cfg_dict.get('use_learnable_rope', False)
             )
             print(f"{model_name} loaded!")
         print("All models loaded!")
@@ -73,13 +71,13 @@ class Examples:
     ----------
     exam_cfg : OmegaConf | DictConfig
         Parsed contents of `examples.yaml`. Expected structure:
-        `examples: [ {text, model, temperature?, top_p?, repetition_penalty?}, ... ]`.
+        `examples: [ {text, model, speaker?, temperature?, top_p?, repetition_penalty?}, ... ]`.
 
     Behavior
     --------
     - Produces a list-of-lists whose order must match the `inputs` order
         used when constructing `gr.Examples` in `app.py`.
-    - Current order: `[text, model_dropdown, temp, top_p, rp]`.
+    - Current order: `[text, model_dropdown, speaker_dropdown, temp, top_p, rp]`.
 
     Why this exists
     ---------------
@@ -95,11 +93,12 @@ class Examples:
         for e in self.exam_cfg.examples:
             text                = e.get("text")
             model               = e.get("model")
+            speaker             = e.get("speaker", "Kore (en)")
             temperature         = e.get("temperature", 1.0)
             top_p               = e.get("top_p", 0.95)
             repetition_penalty  = e.get("repetition_penalty", 1.1)
-            # Order must match gr.Examples inputs: [text, model_dropdown, temp, top_p, rp]
-            rows.append([text, model, temperature, top_p, repetition_penalty])
+            # Order must match gr.Examples inputs: [text, model_dropdown, speaker_dropdown, temp, top_p, rp]
+            rows.append([text, model, speaker, temperature, top_p, repetition_penalty])
 
         return rows