init

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+*test*
+temp

README.md

@@ -1 +1,12 @@
-# VTTS-speechT5
+---
+title: VTTS speecht5
+emoji: 🗣️
+colorFrom: green
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.29.0
+app_file: app.py
+pinned: false
+---
+
+# multilingual_speecht5

app.py

@@ -0,0 +1,47 @@
+import streamlit as st
+st.set_page_config(page_title="SpeechT5", page_icon = "static/images/PLAYGROUND_LOGO_REDESIGN_IMAGE.png")
+hide_menu_style = """
+<style>
+footer {visibility: hidden;}
+</style>
+"""
+st.markdown(hide_menu_style, unsafe_allow_html= True)
+
+import glob
+from src.model import Model, dataset_dict
+
+if "model_name" not in st.session_state:
+    st.session_state.model_name = None
+    st.session_state.audio = None
+    st.session_state.wav_file = None
+    
+with st.sidebar.form("my_form"):
+
+    text = st.text_input("Your input: ")
+    model_name = st.selectbox(label="Model: ", options=["truong-xuan-linh/speecht5-vietnamese-commonvoice", 
+                                                        "truong-xuan-linh/speecht5-vietnamese-voiceclone-lsvsc",
+                                                        "truong-xuan-linh/speecht5-vietnamese-hlpcvoice",
+                                                        "truong-xuan-linh/speecht5-vietnamese-vstnvoice",
+                                                        "truong-xuan-linh/speecht5-vietnamese-kcbnvoice",
+                                                        "truong-xuan-linh/speecht5-irmvivoice",
+                                                        "truong-xuan-linh/speecht5-vietnamese-voiceclone",
+                                                        "truong-xuan-linh/speecht5-multilingual-voiceclone-speechbrain",
+                                                        "truong-xuan-linh/speecht5-vietnamese-voiceclone-v3",
+                                                        "truong-xuan-linh/speecht5-multilingual-voiceclone-pynote",
+                                                        "truong-xuan-linh/speecht5-multilingual-voiceclone-speechbrain-nonverbal"])
+    
+    speaker_id = st.selectbox("source voice", options= list(dataset_dict.keys()))
+    speaker_url = st.text_input("speaker url", value="")
+    # speaker_id = st.selectbox("source voice", options= glob.glob("voices/*.wav"))
+    if st.session_state.model_name != model_name :
+        st.session_state.model_name = model_name
+        st.session_state.model = Model(model_name=model_name)
+        st.session_state.speaker_id = speaker_id
+        
+    # Every form must have a submit button.
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        st.session_state.audio = st.session_state.model.inference(text=text, speaker_id=speaker_id, speaker_url=speaker_url)
+        
+audio_holder = st.empty()
+audio_holder.audio(st.session_state.audio, sample_rate=16000)

requirements.txt

@@ -0,0 +1,11 @@
+torch==2.1.2
+numpy==1.23.5
+transformers==4.38.2
+uroman-python==1.2.8.1
+datasets==2.16.1
+deepfilternet==0.5.6
+torchaudio==2.1.2
+librosa==0.10.0
+streamlit==1.29.0
+pydub==0.25.1
+speechbrain==0.5.16

src/model.py

@@ -0,0 +1,127 @@
+import re
+import torch
+import requests
+import torchaudio
+import numpy as np
+from src.reduce_noise import smooth_and_reduce_noise, model_remove_noise, model, df_state
+import io
+from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
+from pydub import AudioSegment
+import re
+from uroman import uroman
+# from src.pynote_speaker_embedding import create_speaker_embedding
+from src.speechbrain_speaker_embedding import create_speaker_embedding
+
+from datasets import load_dataset
+dataset = load_dataset("truong-xuan-linh/vi-xvector-speechbrain", 
+                       download_mode="force_redownload", 
+                            verification_mode="no_checks", 
+                            cache_dir="temp/",
+                            revision="5ea5e4345258333cbc6d1dd2544f6c658e66a634")
+dataset = dataset["train"].to_list()
+
+dataset_dict = {}
+
+for rc in dataset:
+    dataset_dict[rc["speaker_id"]] = rc["embedding"]
+    
+vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+
+def remove_special_characters(sentence):
+    # Use regular expression to keep only letters, periods, and commas
+    sentence_after_removal =  re.sub(r'[^a-zA-Z\s,.\u00C0-\u1EF9]', ' ', sentence)
+    return sentence_after_removal
+
+from scipy.signal import butter, lfilter
+
+def butter_bandpass(lowcut, highcut, fs, order=5):
+    nyq = 0.5 * fs
+    low = lowcut / nyq
+    high = highcut / nyq
+    b, a = butter(order, [low, high], btype='band')
+    return b, a
+
+def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
+    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
+    y = lfilter(b, a, data)
+    return y
+
+def korean_splitter(string):
+    pattern = re.compile('[가-힣]+')
+    matches = pattern.findall(string)
+    return matches
+
+def uroman_normalization(string):
+    korean_inputs = korean_splitter(string)
+    for korean_input in korean_inputs:
+        korean_roman = uroman(korean_input)
+        string = string.replace(korean_input, korean_roman)
+    return string
+
+class Model():
+    
+    def __init__(self, model_name):
+        self.model_name = model_name
+        self.processor = SpeechT5Processor.from_pretrained(model_name)
+        self.model = SpeechT5ForTextToSpeech.from_pretrained(model_name)
+        # self.model.generate = partial(self.model.generate, use_cache=True)
+
+        self.model.eval()
+        if model_name == "truong-xuan-linh/speecht5-vietnamese-commonvoice" or model_name == "truong-xuan-linh/speecht5-irmvivoice":
+            self.speaker_embeddings = torch.zeros((1, 512))  # or load xvectors from a file
+        else:
+            self.speaker_embeddings = torch.ones((1, 512))  # or load xvectors from a file
+            
+    def inference(self, text, speaker_id=None, speaker_url=""):
+        # if self.model_name == "truong-xuan-linh/speecht5-vietnamese-voiceclone-v2":
+        #     # self.speaker_embeddings = torch.tensor(dataset_dict_v2[speaker_id])
+        #     wavform, _ = torchaudio.load(speaker_id)
+        #     self.speaker_embeddings = create_speaker_embedding(wavform)[0]
+            
+        if "voiceclone" in self.model_name:
+            if not speaker_url:
+                self.speaker_embeddings = torch.tensor(dataset_dict[speaker_id])
+            else:
+                response = requests.get(speaker_url)
+                audio_stream = io.BytesIO(response.content)
+                audio_segment = AudioSegment.from_file(audio_stream, format="wav")
+                audio_segment = audio_segment.set_channels(1)
+                audio_segment = audio_segment.set_frame_rate(16000)
+                audio_segment = audio_segment.set_sample_width(2)
+                wavform, _ = torchaudio.load(audio_segment.export())
+                self.speaker_embeddings = create_speaker_embedding(wavform)[0]
+            # self.speaker_embeddings = create_speaker_embedding(speaker_id)[0]
+            # wavform, _ = torchaudio.load("voices/kcbn1.wav")
+            # self.speaker_embeddings = create_speaker_embedding(wavform)[0]
+            # wavform, _ = torchaudio.load(wav_file)
+            # self.speaker_embeddings = create_speaker_embedding(wavform)[0]
+        
+            
+        with torch.no_grad():
+            full_speech = []
+            separators = r";|\.|!|\?|\n"
+            text = uroman_normalization(text)
+            text = text.replace(" ", "▁")
+            split_texts = re.split(separators, text)
+            
+            for split_text in split_texts:
+                
+                if split_text != "▁":
+                    # split_text = remove_special_characters(" ," + split_text) + " ,"
+                    split_text = split_text.lower() + "▁"
+                    print(split_text)
+                    inputs = self.processor.tokenizer(text=split_text, return_tensors="pt")
+                    speech = self.model.generate_speech(inputs["input_ids"], threshold=0.5, speaker_embeddings=self.speaker_embeddings, vocoder=vocoder)
+                    full_speech.append(speech.numpy())
+                    # full_speech.append(butter_bandpass_filter(speech.numpy(), lowcut=10, highcut=5000, fs=16000, order=2))
+            out_audio = model_remove_noise(model, df_state, np.concatenate(full_speech))
+            return out_audio
+    
+    @staticmethod
+    def moving_average(data, window_size):
+        return np.convolve(data, np.ones(window_size)/window_size, mode='same')
+    
+# woman: VIVOSSPK26, VIVOSSPK02, VIVOSSPK40
+
+# man: VIVOSSPK28, VIVOSSPK36, VIVOSDEV09, VIVOSSPK33, VIVOSSPK23
+

src/pynote_speaker_embedding.py

@@ -0,0 +1,12 @@
+# import torch
+# from pyannote.audio import Model, Inference
+# speaker_model = Model.from_pretrained("pyannote/embedding",
+#                               use_auth_token="")
+# inference = Inference(speaker_model, window="whole")
+
+# def create_speaker_embedding(audio_dir):
+#     with torch.no_grad():
+#         embedding = inference(audio_dir)
+#         embedding = torch.tensor([[embedding]])
+#         speaker_embeddings = torch.nn.functional.normalize(embedding, dim=-1)
+#     return speaker_embeddings

src/reduce_noise.py

@@ -0,0 +1,43 @@
+import scipy.signal
+import librosa
+from df.enhance import enhance, init_df, load_audio, save_audio
+import torch
+from torchaudio.functional import resample
+
+# Load default model
+model, df_state, _ = init_df()
+
+def smooth_and_reduce_noise(audio_signal, sampling_rate):
+    # Apply a low-pass filter for smoothing
+    cutoff_frequency = 1700  # Adjust as needed
+    nyquist = 0.5 * sampling_rate
+    normal_cutoff = cutoff_frequency / nyquist
+    b, a = scipy.signal.butter(4, normal_cutoff, btype='low', analog=False)
+    smoothed_signal = scipy.signal.filtfilt(b, a, audio_signal)
+
+    # Reduce noise using librosa's denoiser
+    denoised_signal = librosa.effects.preemphasis(smoothed_signal, coef=0.95)
+
+    return denoised_signal
+
+def model_remove_noise(model, df_state, np_audio):
+    #Read audio
+    audio = torch.tensor([np_audio])
+    audio = resample(audio, 16000, df_state.sr())
+    
+    #Inference
+    enhanced = enhance(model, df_state, audio).cpu().numpy()
+    
+    #Save
+    dtype=torch.int16
+    out_audio = torch.as_tensor(enhanced)
+    if out_audio.ndim == 1:
+        out_audio.unsqueeze_(0)
+    if dtype == torch.int16 and out_audio.dtype != torch.int16:
+        out_audio = (out_audio * (1 << 15)).to(torch.int16)
+    if dtype == torch.float32 and out_audio.dtype != torch.float32:
+        out_audio = out_audio.to(torch.float32) / (1 << 15)
+        
+    out_audio = resample(audio, df_state.sr(), 16000)
+    
+    return out_audio.cpu().numpy()

src/speechbrain_speaker_embedding.py

@@ -0,0 +1,19 @@
+import torch
+
+import os
+from speechbrain.pretrained import EncoderClassifier
+
+spk_model_name = "speechbrain/spkrec-xvect-voxceleb"
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+speaker_model = EncoderClassifier.from_hparams(
+    source=spk_model_name,
+    run_opts={"device": device},
+    savedir=os.path.join("/tmp", spk_model_name),
+)
+
+def create_speaker_embedding(waveform):
+    with torch.no_grad():
+        speaker_embeddings = speaker_model.encode_batch(waveform)
+        speaker_embeddings = torch.nn.functional.normalize(speaker_embeddings, dim=-1)
+    return speaker_embeddings