Add TeraTTS and remove pyttsx3

.gitignore

@@ -151,7 +151,7 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-
+model/
 results/
 checkpoints/
 gradio_cached_examples/

app.py

@@ -1,12 +1,31 @@
-import os, sys
+import os
 import uuid
-import tempfile
-import pyttsx3
 import gradio as gr
 from src.gradio_demo import SadTalker  
-# from src.utils.text2speech import TTSTalker
+from infer_onnx import TTS
 from huggingface_hub import snapshot_download
 
+
+# Список моделей TTS для выбора
+models = ["TeraTTS/natasha-g2p-vits", "TeraTTS/glados2-g2p-vits", "TeraTTS/glados-g2p-vits", "TeraTTS/girl_nice-g2p-vits"]
+
+# Создаем словарь моделей и инициализируем их
+models = {k: TTS(k) for k in models}
+
+# Функция для синтеза речи
+def text_to_speech(model_name, length_scale, text):
+    time_tag = str(uuid.uuid4())
+    save_dir = './results/voice_input'
+    os.makedirs(save_dir, exist_ok=True)
+    file_name = os.path.join(save_dir, os.path.basename(time_tag + '.wav')) 
+    
+    open(file_name, "wb").close()
+
+    audio = models[model_name](text, length_scale=length_scale)
+    models[model_name].save_wav(audio, file_name, sample_rate=models[model_name].config["samplerate"])
+
+    return file_name
+
 def get_source_image(image):   
         return image
 
@@ -33,92 +52,11 @@ def download_model():
      REPO_ID = 'vinthony/SadTalker-V002rc'
      snapshot_download(REPO_ID)
 
-# language  : en_US, de_DE, ...
-# gender    : VoiceGenderFemale, VoiceGenderMale
-def change_voice(engine, language='ru_ru', gender='male'):
-
-    selected_voices = []
-    
-    language = language.lower() if language else ''
-    gender = gender.lower() if gender else ''
-
-    for voice in engine.getProperty('voices'):
-        voice_appended = False
-
-        for lang in voice.languages:
-            
-            lang_str = str(lang, 'utf-8')
-            print("lang", lang_str)
-            
-            if lang_str and language in lang_str.lower():
-                selected_voices.append(voice)
-                print("voice appended by lang", voice, lang_str)
-                voice_appended = True
-                break
-
-        if voice_appended:
-            continue
-
-        if voice.id and language in voice.id.lower():
-            selected_voices.append(voice)
-            print("voice appended by id", voice.id)
-            continue
-
-        if voice.name and language in voice.name.lower():
-            selected_voices.append(voice)
-            print("voice appended by name", voice.name)
-            continue
-
-    for voice in selected_voices:
-        if voice.gender and gender in voice.gender.lower():
-            engine.setProperty('voice', voice.id)
-            print("voice selected by gender", voice.gender)
-            return True
-    
-        if voice.id and gender in voice.id.lower():
-            engine.setProperty('voice', voice.id)
-            print("voice selected by id", voice.id)
-            return True
-        if voice.name and gender in voice.name.lower():
-            engine.setProperty('voice', voice.id)
-            print("voice selected by name", voice.name)
-            return True
-
-    if len(selected_voices) > 0:
-        engine.setProperty('voice', selected_voices[0].id)
-        print("voice selected by default", selected_voices[0].id)
-        return True
-    
-    return False
-
-def play_text_to_speech(text_input, voice_option):
-    engine = pyttsx3.init()
-    
-    change_voice(engine, 'ru', voice_option)
-
-    print("text_input", text_input)
-    print("voice_option", voice_option)
-   
-    time_tag = str(uuid.uuid4())
-    save_dir = './results/voice_input'
-    os.makedirs(save_dir, exist_ok=True)
-    file_name = os.path.join(save_dir, os.path.basename(time_tag + '.wav')) 
-    
-    open(file_name, "wb").close()
-    engine.say(text_input)
-    engine.save_to_file(text_input, file_name)
-    engine.runAndWait()
-    
-    print("file saved to", file_name)
-
-    return file_name
-
 def sadtalker_demo():
 
     download_model()
 
     sad_talker = SadTalker(lazy_load=True)
-    # tts_talker = TTSTalker()
 
     with gr.Blocks(analytics_enabled=False) as sadtalker_interface: 
         with gr.Row():
@@ -131,6 +69,12 @@ def sadtalker_demo():
 
                 with gr.Tabs(elem_id="sadtalker_driven_audio"):
                     with gr.TabItem('Driving Methods'):
+                        with gr.Row():
+                            model_choice = gr.Dropdown(choices=list(models.keys()), value="TeraTTS/natasha-g2p-vits", label="Choose TTS model")
+                        with gr.Row():
+                            length_scale = gr.Slider(minimum=0.1, maximum=2.0, label="Length scale (increase length of sound) Default: 1.2", value=1.2)
+                        with gr.Row():
+                            input_text = gr.Textbox(label="Enter text")
                         with gr.Row():
                             driven_audio = gr.Audio(label="Input audio", source="upload", type="filepath")
                             driven_audio_no = gr.Audio(label="Use IDLE mode, no audio is required", source="upload", type="filepath", visible=False)
@@ -139,15 +83,10 @@ def sadtalker_demo():
                                 use_idle_mode = gr.Checkbox(label="Use Idle Animation", visible=False)
                                 length_of_audio = gr.Number(value=5, label="The length(seconds) of the generated video.")
                                 use_idle_mode.change(toggle_audio_file, inputs=use_idle_mode, outputs=[driven_audio, driven_audio_no]) # todo
-                        with gr.Row():
-                            text_input = gr.Textbox(label="Enter text", multiline=True)
-                            voice_option = gr.Radio(['Male', 'Female'], label='Voice Option', value='Female')
                         with gr.Row():
                             play_button = gr.Button('Text To Speech', variant='primary')
                             play_button.click(
-                                fn=play_text_to_speech,
-                                inputs=[text_input, voice_option],
-                                outputs=[driven_audio]
+                                fn=text_to_speech, inputs=[model_choice, length_scale, input_text], outputs=[driven_audio]
                             ) 
                         with gr.Row():
                             ref_video = gr.Video(label="Reference Video", source="upload", type="filepath", elem_id="vidref")
@@ -158,13 +97,10 @@ def sadtalker_demo():
 
                             ref_video.change(ref_video_fn, inputs=ref_video, outputs=[use_ref_video]) # todo
 
-
             with gr.Column(variant='panel'): 
                 with gr.Tabs(elem_id="sadtalker_checkbox"):
                     with gr.TabItem('Settings'):
                         with gr.Column(variant='panel'):
-                            # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
-                            # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
                             with gr.Row():
                                 pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
                                 exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) # 
@@ -187,8 +123,6 @@ def sadtalker_demo():
                 with gr.Tabs(elem_id="sadtalker_genearted"):
                         gen_video = gr.Video(label="Generated video", format="mp4")
 
-        
-
         submit.click(
                 fn=sad_talker.test,
                 inputs=[source_image,
@@ -213,7 +147,6 @@ def sadtalker_demo():
 
     return sadtalker_interface
  
-
 if __name__ == "__main__":
 
     demo = sadtalker_demo()

infer_onnx.py

@@ -0,0 +1,90 @@
+import scipy.io.wavfile
+import os
+import onnxruntime
+import numpy as np
+from huggingface_hub import snapshot_download
+from num2words import num2words
+import re
+from transliterate import translit
+import json
+
+class TTS:
+    def __init__(self, model_name: str, save_path: str = "./model", add_time_to_end: float = 0.8) -> None:
+        if not os.path.exists(save_path):
+            os.mkdir(save_path)
+        
+        model_dir = os.path.join(save_path, model_name)
+        
+        if not os.path.exists(model_dir):
+            snapshot_download(repo_id=model_name, 
+                              allow_patterns=["*.txt", "*.onnx", "*.json"], 
+                              local_dir=model_dir,
+                              local_dir_use_symlinks=False
+                            )
+        
+        self.model = onnxruntime.InferenceSession(os.path.join(model_dir, "exported/model.onnx"), providers=['CPUExecutionProvider'])
+        with open(os.path.join(model_dir, "exported/config.json")) as config_file:
+            self.config = json.load(config_file)["model_config"]
+            
+        if os.path.exists(os.path.join(model_dir, "exported/dictionary.txt")):
+            from tokenizer import TokenizerG2P
+            print("Use g2p")
+            self.tokenizer = TokenizerG2P(os.path.join(model_dir, "exported"))
+            
+        else:
+            from tokenizer import TokenizerGRUUT
+            print("Use gruut")
+            self.tokenizer = TokenizerGRUUT(os.path.join(model_dir, "exported"))
+        
+        self.add_time_to_end = add_time_to_end
+
+    
+    def _add_silent(self, audio, silence_duration: float = 1.0, sample_rate: int = 22050):
+        num_samples_silence = int(sample_rate * silence_duration)
+        silence_array = np.zeros(num_samples_silence, dtype=np.float32)
+        audio_with_silence = np.concatenate((audio, silence_array), axis=0)
+        return audio_with_silence
+
+
+    def save_wav(self, audio, path:str, sample_rate: int = 22050):
+        '''save audio to wav'''
+        scipy.io.wavfile.write(path, sample_rate, audio)
+    
+    
+    def _intersperse(self, lst, item):
+        result = [item] * (len(lst) * 2 + 1)
+        result[1::2] = lst
+        return result
+    
+    def _get_seq(self, text):
+        phoneme_ids = self.tokenizer._get_seq(text)
+        phoneme_ids_inter = self._intersperse(phoneme_ids, 0)
+        return phoneme_ids_inter
+        
+    def _num2wordsshor(self, match):
+        match = match.group()
+        ret = num2words(match, lang ='ru')
+        return ret 
+    
+    def __call__(self, text: str, length_scale=1.2):
+        text = translit(text, 'ru')
+        text = re.sub(r'\d+',self._num2wordsshor,text)
+        phoneme_ids = self._get_seq(text)
+        text = np.expand_dims(np.array(phoneme_ids, dtype=np.int64), 0)
+        text_lengths = np.array([text.shape[1]], dtype=np.int64)
+        scales = np.array(
+            [0.667, length_scale, 0.8],
+            dtype=np.float32,
+        )
+        audio = self.model.run(
+            None,
+            {
+                "input": text,
+                "input_lengths": text_lengths,
+                "scales": scales,
+                "sid": None,
+            },
+        )[0][0,0][0]
+        
+        audio = self._add_silent(audio, silence_duration = self.add_time_to_end, sample_rate=self.config["samplerate"])
+        return audio

packages.txt

@@ -1,3 +1,2 @@
 ffmpeg
-libsndfile1
-libespeak1
+libsndfile1

requirements.txt

@@ -22,4 +22,12 @@ dlib-bin
 gfpgan
 av
 safetensors
-pyttsx3==2.90
+gruut
+gruut-lang-ru
+onnxruntime
+huggingface-hub==0.15.1
+transformers
+sentencepiece
+ruaccent
+transliterate
+num2words

tokenizer/__init__.py

@@ -0,0 +1,2 @@
+from .gruut import Tokenizer as TokenizerGRUUT
+from .g2p import Tokenizer as TokenizerG2P

tokenizer/g2p/__init__.py

@@ -0,0 +1 @@
+from .tokenizer import Tokenizer

tokenizer/g2p/g2p.py

@@ -0,0 +1,94 @@
+
+softletters=set(u"яёюиье")
+startsyl=set(u"#ъьаяоёуюэеиы-")
+others = set(["#", "+", "-", u"ь", u"ъ"])
+
+softhard_cons = {
+    u"б" : u"b",
+    u"в" : u"v",
+    u"г" : u"g",
+    u"Г" : u"g",
+    u"д" : u"d",
+    u"з" : u"z",
+    u"к" : u"k",
+    u"л" : u"l",
+    u"м" : u"m",
+    u"н" : u"n",
+    u"п" : u"p",
+    u"р" : u"r",
+    u"с" : u"s",
+    u"т" : u"t",
+    u"ф" : u"f",
+    u"х" : u"h"
+}
+
+other_cons = {
+    u"ж" : u"zh",
+    u"ц" : u"c",
+    u"ч" : u"ch",
+    u"ш" : u"sh",
+    u"щ" : u"sch",
+    u"й" : u"j"
+}
+
+vowels = {
+    u"а" : u"a",
+    u"я" : u"a",
+    u"у" : u"u",
+    u"ю" : u"u",
+    u"о" : u"o",
+    u"ё" : u"o",
+    u"э" : u"e",
+    u"е" : u"e",
+    u"и" : u"i",
+    u"ы" : u"y",
+}                                
+
+def pallatize(phones):
+    for i, phone in enumerate(phones[:-1]):
+        if phone[0] in softhard_cons:
+            if phones[i+1][0] in softletters:
+                phones[i] = (softhard_cons[phone[0]] + "j", 0)
+            else:
+                phones[i] = (softhard_cons[phone[0]], 0)
+        if phone[0] in other_cons:
+            phones[i] = (other_cons[phone[0]], 0)
+
+def convert_vowels(phones):
+    new_phones = []
+    prev = ""
+    for phone in phones:
+        if prev in startsyl:
+            if phone[0] in set(u"яюеё"):
+                new_phones.append("j")
+        if phone[0] in vowels:
+            new_phones.append(vowels[phone[0]] + str(phone[1]))
+        else:
+            new_phones.append(phone[0])
+        prev = phone[0]
+
+    return new_phones
+
+def convert(stressword):
+    phones = ("#" + stressword + "#")
+
+
+    # Assign stress marks
+    stress_phones = []
+    stress = 0
+    for phone in phones:
+        if phone == "+":
+            stress = 1
+        else:
+            stress_phones.append((phone, stress))
+            stress = 0
+
+    # Pallatize
+    pallatize(stress_phones)
+
+    # Assign stress
+    phones = convert_vowels(stress_phones)
+
+    # Filter
+    phones = [x for x in phones if x not in others]
+    return " ".join(phones)

tokenizer/g2p/tokenizer.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+import re
+from .g2p import *
+import json
+import os
+
+class Tokenizer():
+    def __init__(self, data_path: str) -> None:
+        self.dic = {}
+        for line in open(os.path.join(data_path, "dictionary.txt")):
+           items = line.split()
+           self.dic[items[0]] = " ".join(items[1:])
+
+        self.config = json.load(open(os.path.join(data_path, "config.json")))
+    
+    def g2p(self, text):
+        text = re.sub("—", "-", text)
+        text = re.sub("([!'(),-.:;?])", r' \1 ', text)
+
+        phonemes = []
+        for word in text.split():
+            if re.match("[!'(),-.:;?]", word):
+                phonemes.append(word)
+                continue
+
+            word = word.lower()
+            if len(phonemes) > 0: phonemes.append(' ')
+
+            if word in self.dic:
+                phonemes.extend(self.dic[word].split())
+            else:
+                phonemes.extend(convert(word).split())
+
+        phoneme_id_map = self.config["phoneme_id_map"]
+        phoneme_ids = []
+        phoneme_ids.extend(phoneme_id_map["^"])
+        phoneme_ids.extend(phoneme_id_map["_"])
+        for p in phonemes:
+            if p in phoneme_id_map:
+                phoneme_ids.extend(phoneme_id_map[p])
+                phoneme_ids.extend(phoneme_id_map["_"])
+        phoneme_ids.extend(phoneme_id_map["$"])
+
+        return phoneme_ids, phonemes
+    
+    def _get_seq(self, text: str) -> list[int]:
+        seq = self.g2p(text)[0]
+        return seq

tokenizer/gruut/__init__.py

@@ -0,0 +1 @@
+from .tokenizer import Tokenizer

tokenizer/gruut/tokenizer.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+from gruut import sentences
+import os
+import re
+
+class Tokenizer():
+    def __init__(self, path) -> None:
+        with open(os.path.join(path, "vocab.txt"), "r", encoding="utf-8") as vocab_file:
+            self.symbols = vocab_file.read().split("\n")
+            self.symbols = list(map(chr, list(map(int, self.symbols))))
+            
+        self.symbol_to_id = {s: i for i, s in enumerate(self.symbols)}
+        
+    def _ru_phonems(self, text: str) -> str:
+        text = text.lower()
+        phonemes = ""
+        for sent in sentences(text, lang="ru"):
+            for word in sent:
+                if word.phonemes:
+                    phonemes += "".join(word.phonemes)
+        phonemes = re.sub(re.compile(r'\s+'), ' ', phonemes).lstrip().rstrip()
+        return phonemes
+    
+    
+    def _text_to_sequence(self, text: str) -> list[int]:
+        '''convert text to seq'''
+        sequence = []
+        clean_text = self._ru_phonems(text)
+        for symbol in clean_text:
+            symbol_id = self.symbol_to_id[symbol]
+            sequence += [symbol_id]
+        return sequence
+    
+    
+    def _get_seq(self, text: str) -> list[int]:
+        seq = self._text_to_sequence(text)
+        return seq