Update ASR engine to whisper based

.gitignore

@@ -1,7 +1,7 @@
 flagged
-wav
 samples
-wav
+wav/*.wav
+wav/**/*.wav
 wav.bak
 
 model

app.py

@@ -15,8 +15,7 @@ from pathlib import Path
 # local import
 import sys
 from espnet2.bin.tts_inference import Text2Speech
-
-# pdb.set_trace()
+from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModelForCTC# pdb.set_trace()
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 
 sys.path.append("src")
@@ -34,10 +33,22 @@ audio_files = [
     )
 ]
 # audio_files = [str(x) for x in sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
-transcriber = pipeline(
-    "automatic-speech-recognition",
-    model="KevinGeng/PAL_John_128_train_dev_test_seed_1",
-)
+# transcriber = pipeline(
+#     "automatic-speech-recognition",
+#     model="KevinGeng/PAL_John_128_train_dev_test_seed_1",
+# )
+
+from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
+
+processor = AutoProcessor.from_pretrained("KevinGeng/whipser_medium_en_PAL300_step25")
+
+model = AutoModelForSpeechSeq2Seq.from_pretrained("KevinGeng/whipser_medium_en_PAL300_step25")
+
+# feature_extractor = AutoFeatureExtractor.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# representation_model = AutoModelForCTC.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# tokenizer = AutoTokenizer.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/whipser_medium_en_PAL300_step25")
 # transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
 # 【Female】kan-bayashi ljspeech parallel wavegan
 # tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
@@ -211,9 +222,6 @@ def download_file(audio_file):
     return gr.File(value=audio_file)
 # pdb.set_trace()
 
-# if __name__ == "__main__":
-#     file_share_app.run(port=3000)    
-
 with gr.Blocks(
     analytics_enabled=False,
     css=".gradio-container {background-color: #78BD91}",
@@ -249,7 +257,7 @@ with gr.Blocks(
     b2 = gr.Button("Convert")
     
     output_audio = gr.Audio(
-        source="upload", label="Converted Audio", interactive=False
+        source="upload", file="filepath", label="Converted Audio", interactive=False
     )
     
     b2.click(
@@ -258,5 +266,7 @@ with gr.Blocks(
         outputs=output_audio,
         api_name="convert"
     )
+    
+# download_file("wav/001_F1_spkembs.wav")
 
 demo.launch(share=False)

app.ver1.py

@@ -0,0 +1,72 @@
+#TODO:
+#  + [x] Load Configuration
+#  + [ ] Checking
+#  + [ ] Better saving directory
+
+from pathlib import Path
+from transformers import pipeline
+import torch.nn as nn
+import torch
+import torchaudio
+import gradio as gr
+import sys
+
+# Local imports
+sys.path.append("src")
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+# Check if GPU is available
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# ASR part
+
+data_path = "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+audio_files = sorted(list(Path(data_path).glob("**/*wav")))
+# audio_files = sorted(list(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav")))
+
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+
+# TTS part
+def load_model(lang, tag, vocoder_tag):
+    if lang == "Japanese":
+        if tag == "kan-bayashi/ljspeech_parallel_wavegan":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/ljspeech_parallel_wavegan")
+        elif tag == "kan-bayashi/ljspeech_merlin_multi_band_melgan":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/ljspeech_merlin_multi_band_melgan")
+        else:
+            raise ValueError(f"Not supported: lang={lang}, tag={tag}")
+        vocoder = None if vocoder_tag == "none" else vocoder_tag
+    elif lang == "English":
+        # VITS needs no vocoder; others do
+        if tag == "kan-bayashi/libritts_xvector_vits":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/libritts_xvector_vits")
+            vocoder = None
+        elif tag == "kan-bayashi/fastspeech2_en_libritts_guessspeaker_melgan.v3":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/fastspeech2_en_libritts_guessspeaker_melgan.v3")
+            vocoder = "melgan"
+        else:
+            raise ValueError(f"Not supported: lang={lang}, tag={tag}")
+    else:
+        raise ValueError(f"Not supported: lang={lang}")
+    return tts_model, vocoder
+
+tts_model, vocoder_tag = load_model(lang="English", tag="kan-bayashi/libritts_xvector_vits", vocoder_tag="parallel_wavegan/vctk_parallel_wavegan.v1.long")
+tts_model = tts_model.to(device)
+
+vocoder = None if vocoder_tag == "none" else torchaudio.models.vocoder.from_pretrained(vocoder_tag).to(device)
+
+# Gradio part
+def synthesize(text):
+    with torch.no_grad():
+        # Text-to-speech
+        wav = tts_model(text)[0]
+        if vocoder is not None:
+            # Apply vocoder
+            wav = vocoder.inference(wav)
+        # Convert to numpy array
+        wav = wav.squeeze().cpu().numpy()
+    return wav
+
+interface = gr.Interface(synthesize, inputs="text", outputs="audio")
+interface.launch()

app.whisper.fine_tuned.py

@@ -0,0 +1,272 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from pathlib import Path
+
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModelForCTC# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [
+    str(x)
+    for x in sorted(
+        Path(
+            "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+        ).glob("**/*wav")
+    )
+]
+# audio_files = [str(x) for x in sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+# transcriber = pipeline(
+#     "automatic-speech-recognition",
+#     model="KevinGeng/PAL_John_128_train_dev_test_seed_1",
+# )
+
+from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
+
+processor = AutoProcessor.from_pretrained("KevinGeng/whipser_medium_en_PAL300_step25")
+
+model = AutoModelForSpeechSeq2Seq.from_pretrained("KevinGeng/whipser_medium_en_PAL300_step25")
+
+# feature_extractor = AutoFeatureExtractor.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# representation_model = AutoModelForCTC.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# tokenizer = AutoTokenizer.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/whipser_medium_en_PAL300_step25")
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+
+# @title English multi-speaker pretrained model { run: "auto" }
+lang = "English"
+tag = "kan-bayashi/libritts_xvector_vits"
+# vits needs no
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [
+    p
+    for p in glob.glob(
+       f"xvector/test-clean/spk_xvector.ark", recursive=True
+    )
+    if "test" in p
+][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+spks = list(xvectors.keys())
+
+male_spks = {
+    "Male1": "2300_131720",
+    "Male2": "1320_122612",
+}
+    # "M3": "1188_133604",
+    # "M4": "61_70970",
+female_spks = {"Female1": "2961_961", "Female2": "8463_287645", }
+# "F3": "121_121726"
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+
+## 20230224 Mousa: No reference,
+def ASRTTS(audio_file, spk_name, ref_text=""):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+    if ref_text == "":
+        reg_text = transcriber(audio_file)["text"]
+    else:
+        reg_text = ref_text
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = text2speech(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+
+    return save_id, reg_text
+
+
+def ASRTTS_clean(audio_file, spk_name):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+
+    reg_text = transcriber(audio_file)["text"]
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = text2speech(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+    return save_id
+
+
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+
+input_audio = gr.Audio(
+    source="upload", type="filepath", label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+    source="upload", file="filepath", label="Synthesized Audio"
+)
+examples = [
+    ["./samples/001.wav", "M1", ""],
+    ["./samples/002.wav", "M2", ""],
+    ["./samples/003.wav", "F1", ""],
+    ["./samples/004.wav", "F2", ""],
+]
+
+
+def change_audiobox(choice):
+    if choice == "upload":
+        input_audio = gr.Audio.update(source="upload", visible=True)
+    elif choice == "microphone":
+        input_audio = gr.Audio.update(source="microphone", visible=True)
+    else:
+        input_audio = gr.Audio.update(visible=False)
+    return input_audio
+
+
+def show_icon(choice):
+    if choice == "Male1":
+        spk_icon = gr.Image.update(value="speaker_icons/male1.png", visible=True)
+    elif choice == "Male2":
+        spk_icon = gr.Image.update(value="speaker_icons/male2.png", visible=True)
+    elif choice == "Female1":
+        spk_icon = gr.Image.update(value="speaker_icons/female1.png", visible=True)
+    elif choice == "Female2":
+        spk_icon = gr.Image.update(value="speaker_icons/female2.png", visible=True)
+    return spk_icon
+
+def get_download_file(audio_file=None):
+    if audio_file == None:
+        output_audio_file = gr.File.update(visible=False)
+    else:
+        output_audio_file = gr.File.update(visible=True)
+    return output_audio_file
+        
+def download_file(audio_file):
+    return gr.File(value=audio_file)
+# pdb.set_trace()
+
+with gr.Blocks(
+    analytics_enabled=False,
+    css=".gradio-container {background-color: #78BD91}",
+) as demo:
+    with gr.Column(elem_id="Column"):
+        input_format = gr.Radio(
+            choices=["microphone", "upload"], label="Choose your input format", elem_id="input_format"
+        )
+        input_audio = gr.Audio(
+            source="microphone",
+            type="filepath",
+            label="Input Audio",
+            interactive=True,
+            visible=False,
+            elem_id="input_audio"
+        )
+        input_format.change(
+            fn=change_audiobox, inputs=input_format, outputs=input_audio
+        )
+
+        speaker_option = gr.Radio(choices=spk_names, value="Male1", label="Choose your voice profile")
+        spk_icon = gr.Image(value="speaker_icons/male1.png",
+                            type="filepath",
+                            image_mode="RGB",
+                            source="upload",
+                            shape=[50, 50],
+                            interactive=True,
+                            visible=True)
+        speaker_option.change(
+            fn=show_icon, inputs=speaker_option, outputs=spk_icon
+        )
+
+    b2 = gr.Button("Convert")
+    
+    output_audio = gr.Audio(
+        source="upload", file="filepath", label="Converted Audio", interactive=False
+    )
+    
+    b2.click(
+        ASRTTS_clean,
+        inputs=[input_audio, speaker_option],
+        outputs=output_audio,
+        api_name="convert"
+    )
+    
+# download_file("wav/001_F1_spkembs.wav")
+
+demo.launch(share=False)

local/ASR_compare.py

@@ -44,6 +44,26 @@ transcriber = pipeline(
 old_transcriber = pipeline(
     "automatic-speech-recognition", "facebook/wav2vec2-base-960h"
 )
+whisper_transcriber = pipeline(
+    "automatic-speech-recognition", "KevinGeng/whipser_medium_en_PAL300_step25"
+)
+
+whisper_transcriber_org =  pipeline(
+    "automatic-speech-recognition", "KevinGeng/whisper-medium-PAL128-25step"
+)
+
+whisper_transcriber_Tony  = pipeline(
+    "automatic-speech-recognition", "KevinGeng/Tony1_AVA_script_conv_train_conv_dev"
+)
+
+whisper_transcriber_John =  pipeline(
+    "automatic-speech-recognition", "KevinGeng/whipser_medium_en_PAL300_step25_step2_VTCK"
+)
+
+whisper_transcriber_Negel =  pipeline(
+    "automatic-speech-recognition", "KevinGeng/Negel_152_AVA_script_conv_train_conv_dev"
+)
+
 # transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
 # 【Female】kan-bayashi ljspeech parallel wavegan
 # tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
@@ -81,7 +101,7 @@ from espnet_model_zoo.downloader import ModelDownloader
 
 d = ModelDownloader()
 model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
-
+pdb.set_trace()
 # Speaker x-vector selection
 
 xvector_ark = [
@@ -92,6 +112,7 @@ xvector_ark = [
     if "tr" in p
 ][0]
 xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+
 spks = list(xvectors.keys())
 
 male_spks = {
@@ -115,6 +136,25 @@ def ASRnew(audio_file):
     reg_text = transcriber(audio_file)["text"]
     return reg_text
 
+def ASRwhipser_FT(audio_file):
+    reg_text = whisper_transcriber(audio_file)["text"]
+    return reg_text
+
+def ASRwhipser_Org(audio_file):
+    reg_text = whisper_transcriber_org(audio_file)["text"]
+    return reg_text
+
+def ASRwhipser_Tony(audio_file):
+    reg_text = whisper_transcriber_Tony(audio_file)["text"]
+    return reg_text
+
+def ASRwhipser_Negel(audio_file):
+    reg_text = whisper_transcriber_Negel(audio_file)["text"]
+    return reg_text
+
+def ASRwhipser_John(audio_file):
+    reg_text = whisper_transcriber_John(audio_file)["text"]
+    return reg_text
 
 # def ref_reg_callback(audio_file, spk_name, ref_text):
 #     reg_text = ref_text
@@ -190,25 +230,69 @@ with gr.Blocks(
 
     with gr.Row():
         b1 = gr.Button("Conventional Speech Recognition Engine")
-        old_recognization_textbox = gr.Textbox(
+        t1 = gr.Textbox(
             value="",
             placeholder="Recognition output",
             label="Convertional",
         )
         b1.click(
-            ASRold, inputs=[input_audio], outputs=old_recognization_textbox
+            ASRold, inputs=[input_audio], outputs=t1
         )
 
     with gr.Row():
-        b2 = gr.Button("Laronix Speech Recognition Engine")
-        new_recognization_textbox = gr.Textbox(
+        b2 = gr.Button("Laronix Speech Recognition Engine (Ver1, wav2vec2.0+CTC)")
+        t2 = gr.Textbox(
             value="",
             placeholder="Recognition output",
             label="Purposed",
         )
 
         b2.click(
-            ASRnew, inputs=[input_audio], outputs=new_recognization_textbox
+            ASRnew, inputs=[input_audio], outputs=t2
+        )
+    with gr.Row():
+        b3 = gr.Button("Laronix Speech Recognition Engine (Ver2, Whipser)")
+        t3 = gr.Textbox(
+            value="",
+            placeholder="Recognition output",
+            label="Purposed",
+        )
+
+        b3.click(
+            ASRwhipser_FT, inputs=[input_audio], outputs=t3
+        )
+    with gr.Row():
+        b4 = gr.Button("Laronix Speech Recognition Engine (Whipser, FT with Tony)")
+        t4 = gr.Textbox(
+            value="",
+            placeholder="Recognition output",
+            label="Purposed",
+        )
+
+        b4.click(
+            ASRwhipser_Tony, inputs=[input_audio], outputs=t4
+        )
+    with gr.Row():
+        b5 = gr.Button("Laronix Speech Recognition Engine (Whipser, FT with John)")
+        t5 = gr.Textbox(
+            value="",
+            placeholder="Recognition output",
+            label="Purposed",
+        )
+
+        b5.click(
+            ASRwhipser_John, inputs=[input_audio], outputs=t5
+        )
+    with gr.Row():
+        b6 = gr.Button("Laronix Speech Recognition Engine (Whipser, FT with Negel)")
+        t6 = gr.Textbox(
+            value="",
+            placeholder="Recognition output",
+            label="Purposed",
+        )
+
+        b6.click(
+            ASRwhipser_Negel, inputs=[input_audio], outputs=t6
         )
 
 demo.launch(share=True)

local/ASR_conpare.py

@@ -0,0 +1,72 @@
+#TODO:
+#  + [x] Load Configuration
+#  + [ ] Checking
+#  + [ ] Better saving directory
+
+from pathlib import Path
+from transformers import pipeline
+import torch.nn as nn
+import torch
+import torchaudio
+import gradio as gr
+import sys
+
+# Local imports
+sys.path.append("src")
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+# Check if GPU is available
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# ASR part
+
+data_path = "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+audio_files = sorted(list(Path(data_path).glob("**/*wav")))
+# audio_files = sorted(list(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav")))
+
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+
+# TTS part
+def load_model(lang, tag, vocoder_tag):
+    if lang == "Japanese":
+        if tag == "kan-bayashi/ljspeech_parallel_wavegan":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/ljspeech_parallel_wavegan")
+        elif tag == "kan-bayashi/ljspeech_merlin_multi_band_melgan":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/ljspeech_merlin_multi_band_melgan")
+        else:
+            raise ValueError(f"Not supported: lang={lang}, tag={tag}")
+        vocoder = None if vocoder_tag == "none" else vocoder_tag
+    elif lang == "English":
+        # VITS needs no vocoder; others do
+        if tag == "kan-bayashi/libritts_xvector_vits":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/libritts_xvector_vits")
+            vocoder = None
+        elif tag == "kan-bayashi/fastspeech2_en_libritts_guessspeaker_melgan.v3":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/fastspeech2_en_libritts_guessspeaker_melgan.v3")
+            vocoder = "melgan"
+        else:
+            raise ValueError(f"Not supported: lang={lang}, tag={tag}")
+    else:
+        raise ValueError(f"Not supported: lang={lang}")
+    return tts_model, vocoder
+
+tts_model, vocoder_tag = load_model(lang="English", tag="kan-bayashi/libritts_xvector_vits", vocoder_tag="parallel_wavegan/vctk_parallel_wavegan.v1.long")
+tts_model = tts_model.to(device)
+
+vocoder = None if vocoder_tag == "none" else torchaudio.models.vocoder.from_pretrained(vocoder_tag).to(device)
+
+# Gradio part
+def synthesize(text):
+    with torch.no_grad():
+        # Text-to-speech
+        wav = tts_model(text)[0]
+        if vocoder is not None:
+            # Apply vocoder
+            wav = vocoder.inference(wav)
+        # Convert to numpy array
+        wav = wav.squeeze().cpu().numpy()
+    return wav
+
+interface = gr.Interface(synthesize, inputs="text", outputs="audio")
+interface.launch()

local/PAL_dataset.py

@@ -0,0 +1,34 @@
+## ADD dataset appendning
+from datasets import load_dataset, Dataset, DatasetDict, concatenate_datasets
+import pdb
+
+import numpy as np
+# to_dataset = load_dataset("KevinGeng/testdataset")
+base_dataset = load_dataset("../laronix_automos/data/Patient_sil_trim_16k_normed_5_snr_40")
+base_extra_dataset = load_dataset("../laronix_automos/data/John_p326_large")
+
+PAL_dataset = DatasetDict({"base": base_dataset['train'], "base_extra": base_extra_dataset['train']})
+# PAL_dataset.push_to_hub("KevinGeng/PAL_dataset")
+concatenate_datasets(base_dataset['train'], base_extra_dataset['train'])
+pdb.set_trace()
+
+new_record = {"audio": 
+    {'path': 'Arthur_set1_001_noisy.wav',
+     'array': np.array([0.02526855, 0.04602051, 0.04873657, 0.00045776, 0.00201416, 0.00167847]),
+     'sampling_rate': 16000},
+    "transcription": "TOD"}
+pdb.set_trace()
+
+import requests
+headers = {"Authorization": f"KevinGeng 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"}
+# headers = {"Authorization": "Haopeng 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"}
+# pdb.set_trace()
+API_URL = "https://datasets-server.huggingface.co/is-valid?dataset=KevinGeng/testdataset"
+
+def query():
+    response = requests.request("GET", API_URL, headers=headers)
+    # pdb.set_trace()
+    return response.json()
+data = query()
+
+pdb.set_trace()

local/app.genie.py

@@ -0,0 +1,74 @@
+#TODO:
+#  + [x] Load Configuration
+#  + [ ] Checking
+#  + [ ] Better saving directory
+
+from pathlib import Path
+from transformers import pipeline
+import torch.nn as nn
+import torch
+import torchaudio
+import gradio as gr
+import sys
+
+# Local imports
+sys.path.append("src")
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+# Check if GPU is available
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# ASR part
+
+data_path = "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+audio_files = sorted(list(Path(data_path).glob("**/*wav")))
+# audio_files = sorted(list(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav")))
+
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+
+# TTS part
+def load_model(lang, tag, vocoder_tag):
+    if lang == "Japanese":
+        if tag == "kan-bayashi/ljspeech_parallel_wavegan":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/ljspeech_parallel_wavegan")
+        elif tag == "kan-bayashi/ljspeech_merlin_multi_band_melgan":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/ljspeech_merlin_multi_band_melgan")
+        else:
+            raise ValueError(f"Not supported: lang={lang}, tag={tag}")
+        vocoder = None if vocoder_tag == "none" else vocoder_tag
+    elif lang == "English":
+        # VITS needs no vocoder; others do
+        if tag == "kan-bayashi/libritts_xvector_vits":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/libritts_xvector_vits")
+            vocoder = None
+        elif tag == "kan-bayashi/fastspeech2_en_libritts_guessspeaker_melgan.v3":
+            tts_model = Text2Speech.from_pretrained("kan-bayashi/fastspeech2_en_libritts_guessspeaker_melgan.v3")
+            vocoder = "melgan"
+        else:
+            raise ValueError(f"Not supported: lang={lang}, tag={tag}")
+    else:
+        raise ValueError(f"Not supported: lang={lang}")
+    return tts_model, vocoder
+
+tts_model, vocoder_tag = load_model(lang="English", tag="kan-bayashi/libritts_xvector_vits", vocoder_tag="parallel_wavegan/vctk_parallel_wavegan.v1.long")
+import pdb
+pdb.set_trace()
+tts_model = tts_model.to(device)
+
+vocoder = None if vocoder_tag == "none" else torchaudio.models.vocoder.from_pretrained(vocoder_tag).to(device)
+
+# Gradio part
+def synthesize(text):
+    with torch.no_grad():
+        # Text-to-speech
+        wav = tts_model(text)[0]
+        if vocoder is not None:
+            # Apply vocoder
+            wav = vocoder.inference(wav)
+        # Convert to numpy array
+        wav = wav.squeeze().cpu().numpy()
+    return wav
+
+interface = gr.Interface(synthesize, inputs="text", outputs="audio")
+interface.launch()

local/app.old.py

@@ -0,0 +1,149 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import jiwer
+import pdb
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from time import process_time, time
+from pathlib import Path
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [str(x) for x in  sorted(Path("/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video").glob("**/*wav"))]
+# audio_files = [str(x) for x in  sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+#@title English multi-speaker pretrained model { run: "auto" }
+lang = 'English'
+tag = 'kan-bayashi/libritts_xvector_vits' 
+# tag = "kan-bayashi/vctk_multi_spk_vits"
+# vits needs no 
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long" #@param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [p for p in glob.glob(f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True) if "tr" in p][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+import pdb
+
+pdb.set_trace()
+
+spks = list(xvectors.keys())
+
+male_spks = {"M1": "2300_131720", "M2": "1320_122612", "M3": "1188_133604", "M4": "61_70970"}
+female_spks = {"F1": "2961_961", "F2": "8463_287645", "F3": "121_121726"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+def ASRTTS(audio_file, spk_name, ref_text=""):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+    if ref_text == "":
+        reg_text = transcriber(audio_file)['text']
+    else:
+        reg_text = ref_text
+    
+    speech, sr = torchaudio.load(audio_file, channels_first=True) # Mono channel
+    wav_tensor_spembs = text2speech(text=reg_text, speech=speech, spembs=spembs)["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = "./wav/" + Path(audio_file).stem + "_" + spk_name +"_spkembs.wav"
+    torchaudio.save(save_id, src=wav_tensor_spembs.unsqueeze(0).to("cpu"), sample_rate=22050)
+
+    return save_id, reg_text
+
+def ref_reg_callback(audio_file, spk_name, ref_text):
+    reg_text = ref_text
+    return audio_file, spk_name, reg_text
+    
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+
+input_audio = gr.Audio(
+            source="microphone",
+            type="filepath",
+            label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+        source="upload",
+        file="filepath",
+        label="Synthesized Audio"
+)
+examples = [["./samples/001.wav",'M1', ""],
+            ["./samples/002.wav",'M2', ""],
+            ["./samples/003.wav",'F1', ""],
+            ["./samples/004.wav",'F2', ""]]
+
+# ASRTTS(*examples[0])
+iface = gr.Interface(
+    fn = ASRTTS, 
+    inputs = [
+        input_audio,
+        speaker_option,
+        reference_textbox,
+    ],
+    outputs = [
+        output_audio,
+        recognization_textbox
+    ],
+    examples = examples
+)
+iface.input_callback = ref_reg_callback
+iface.launch(share=False)

local/app.old.whipser.fined_tuned.py

@@ -0,0 +1,146 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import jiwer
+import pdb
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from time import process_time, time
+from pathlib import Path
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [str(x) for x in  sorted(Path("/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video").glob("**/*wav"))]
+# audio_files = [str(x) for x in  sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/whipser_medium_en_PAL300_step25")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+#@title English multi-speaker pretrained model { run: "auto" }
+lang = 'English'
+tag = 'kan-bayashi/libritts_xvector_vits' 
+# tag = "kan-bayashi/vctk_multi_spk_vits"
+# vits needs no 
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long" #@param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [p for p in glob.glob(f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True) if "tr" in p][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+spks = list(xvectors.keys())
+
+male_spks = {"M1": "2300_131720", "M2": "1320_122612", "M3": "1188_133604", "M4": "61_70970"}
+female_spks = {"F1": "2961_961", "F2": "8463_287645", "F3": "121_121726"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+def ASRTTS(audio_file, spk_name, ref_text=""):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+    if ref_text == "":
+        reg_text = transcriber(audio_file)['text']
+    else:
+        reg_text = ref_text
+    
+    speech, sr = torchaudio.load(audio_file, channels_first=True) # Mono channel
+    wav_tensor_spembs = text2speech(text=reg_text, speech=speech, spembs=spembs)["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = "./wav/" + Path(audio_file).stem + "_" + spk_name +"_spkembs.wav"
+    torchaudio.save(save_id, src=wav_tensor_spembs.unsqueeze(0).to("cpu"), sample_rate=22050)
+
+    return save_id, reg_text
+
+def ref_reg_callback(audio_file, spk_name, ref_text):
+    reg_text = ref_text
+    return audio_file, spk_name, reg_text
+    
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+
+input_audio = gr.Audio(
+            source="microphone",
+            type="filepath",
+            label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+        source="upload",
+        file="filepath",
+        label="Synthesized Audio"
+)
+examples = [["./samples/001.wav",'M1', ""],
+            ["./samples/002.wav",'M2', ""],
+            ["./samples/003.wav",'F1', ""],
+            ["./samples/004.wav",'F2', ""]]
+
+# ASRTTS(*examples[0])
+iface = gr.Interface(
+    fn = ASRTTS, 
+    inputs = [
+        input_audio,
+        speaker_option,
+        reference_textbox,
+    ],
+    outputs = [
+        output_audio,
+        recognization_textbox
+    ],
+    examples = examples
+)
+iface.input_callback = ref_reg_callback
+iface.launch(share=False)

local/app.vctk.py

@@ -0,0 +1,146 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import jiwer
+import pdb
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from time import process_time, time
+from pathlib import Path
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [str(x) for x in  sorted(Path("/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video").glob("**/*wav"))]
+# audio_files = [str(x) for x in  sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+#@title English multi-speaker pretrained model { run: "auto" }
+lang = 'English'
+tag = 'kan-bayashi/libritts_xvector_vits' 
+# tag = "kan-bayashi/vctk_multi_spk_vits"
+# vits needs no 
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long" #@param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [p for p in glob.glob(f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True) if "tr" in p][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+spks = list(xvectors.keys())
+
+male_spks = {"M1": "2300_131720", "M2": "1320_122612", "M3": "1188_133604", "M4": "61_70970"}
+female_spks = {"F1": "2961_961", "F2": "8463_287645", "F3": "121_121726"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+def ASRTTS(audio_file, spk_name, ref_text=""):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+    if ref_text == "":
+        reg_text = transcriber(audio_file)['text']
+    else:
+        reg_text = ref_text
+    
+    speech, sr = torchaudio.load(audio_file, channels_first=True) # Mono channel
+    wav_tensor_spembs = text2speech(text=reg_text, speech=speech, spembs=spembs)["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = "./wav/" + Path(audio_file).stem + "_" + spk_name +"_spkembs.wav"
+    torchaudio.save(save_id, src=wav_tensor_spembs.unsqueeze(0).to("cpu"), sample_rate=22050)
+
+    return save_id, reg_text
+
+def ref_reg_callback(audio_file, spk_name, ref_text):
+    reg_text = ref_text
+    return audio_file, spk_name, reg_text
+    
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+
+input_audio = gr.Audio(
+            source="microphone",
+            type="filepath",
+            label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+        source="upload",
+        file="filepath",
+        label="Synthesized Audio"
+)
+examples = [["./samples/001.wav",'M1', ""],
+            ["./samples/002.wav",'M2', ""],
+            ["./samples/003.wav",'F1', ""],
+            ["./samples/004.wav",'F2', ""]]
+
+# ASRTTS(*examples[0])
+iface = gr.Interface(
+    fn = ASRTTS, 
+    inputs = [
+        input_audio,
+        speaker_option,
+        reference_textbox,
+    ],
+    outputs = [
+        output_audio,
+        recognization_textbox
+    ],
+    examples = examples
+)
+iface.input_callback = ref_reg_callback
+iface.launch(share=False)

local/app.whisper.py

@@ -0,0 +1,281 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from pathlib import Path
+
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModelForCTC# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [
+    str(x)
+    for x in sorted(
+        Path(
+            "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+        ).glob("**/*wav")
+    )
+]
+# audio_files = [str(x) for x in sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+# transcriber = pipeline(
+#     "automatic-speech-recognition",
+#     model="KevinGeng/PAL_John_128_train_dev_test_seed_1",
+# )
+
+from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
+
+processor = AutoProcessor.from_pretrained("openai/whisper-medium")
+
+model = AutoModelForSpeechSeq2Seq.from_pretrained("openai/whisper-medium")
+
+# feature_extractor = AutoFeatureExtractor.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# representation_model = AutoModelForCTC.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# tokenizer = AutoTokenizer.from_pretrained("KevinGeng/PAL_John_128_train_dev_test_seed_1")
+
+import pdb
+# pdb.set_trace()
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+
+# @title English multi-speaker pretrained model { run: "auto" }
+lang = "English"
+tag = "kan-bayashi/libritts_xvector_vits"
+# vits needs no
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [
+    p
+    for p in glob.glob(
+       f"xvector/test-clean/spk_xvector.ark", recursive=True
+    )
+    if "test" in p
+][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+spks = list(xvectors.keys())
+
+# pdb.set_trace()
+# All old 20230101
+# male_spks = {"Male1": "2300_131720", "Male2": "1320_122612", "Male3": "1188_133604",}
+    # "M4": "61_70970",
+# female_spks = {"Female1": "2961_961", "Female2": "8463_287645", "Female3": "121_121726"}
+
+# 6 scale from high to low, 
+male_spks = {"Male1": "4077_13751", "Male2": "1320_122612", "Male3": "7729_102255",}
+female_spks = {"Female1": "5683_32865", "Female2": "121_121726", "Female3": "8463_287645"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+
+## 20230224 Mousa: No reference,
+def ASRTTS(audio_file, spk_name, ref_text=""):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+    if ref_text == "":
+        reg_text = transcriber(audio_file)["text"]
+    else:
+        reg_text = ref_text
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = text2speech(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+
+    return save_id, reg_text
+
+
+def ASRTTS_clean(audio_file, spk_name):
+    spk = spks[spk_name]
+    spembs = xvectors[spk]
+
+    reg_text = transcriber(audio_file)["text"]
+
+    speech, sr = torchaudio.load(
+        audio_file, channels_first=True
+    )  # Mono channel
+    wav_tensor_spembs = text2speech(
+        text=reg_text, speech=speech, spembs=spembs
+    )["wav"]
+    wav_numpy = wav_tensor_spembs.unsqueeze(1).to("cpu")
+    sample_rate = 22050
+    save_id = (
+        "./wav/" + Path(audio_file).stem + "_" + spk_name + "_spkembs.wav"
+    )
+    torchaudio.save(
+        save_id,
+        src=wav_tensor_spembs.unsqueeze(0).to("cpu"),
+        sample_rate=22050,
+    )
+    return save_id
+
+
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+
+input_audio = gr.Audio(
+    source="upload", type="filepath", label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+    source="upload", file="filepath", label="Synthesized Audio"
+)
+examples = [
+    ["./samples/001.wav", "M1", ""],
+    ["./samples/002.wav", "M2", ""],
+    ["./samples/003.wav", "F1", ""],
+    ["./samples/004.wav", "F2", ""],
+]
+
+
+def change_audiobox(choice):
+    if choice == "upload":
+        input_audio = gr.Audio.update(source="upload", visible=True)
+    elif choice == "microphone":
+        input_audio = gr.Audio.update(source="microphone", visible=True)
+    else:
+        input_audio = gr.Audio.update(visible=False)
+    return input_audio
+
+
+def show_icon(choice):
+    if choice == "Male1":
+        spk_icon = gr.Image.update(value="speaker_icons/male1.png", visible=True)
+    elif choice == "Male2":
+        spk_icon = gr.Image.update(value="speaker_icons/male2.png", visible=True)
+    elif choice == "Male3":
+        spk_icon = gr.Image.update(value="speaker_icons/male3.png", visible=True)
+    elif choice == "Female1":
+        spk_icon = gr.Image.update(value="speaker_icons/female1.png", visible=True)
+    elif choice == "Female2":
+        spk_icon = gr.Image.update(value="speaker_icons/female2.png", visible=True)
+    elif choice == "Female3":
+        spk_icon = gr.Image.update(value="speaker_icons/female3.png", visible=True)
+    return spk_icon
+
+def get_download_file(audio_file=None):
+    if audio_file == None:
+        output_audio_file = gr.File.update(visible=False)
+    else:
+        output_audio_file = gr.File.update(visible=True)
+    return output_audio_file
+        
+def download_file(audio_file):
+    return gr.File(value=audio_file)
+# pdb.set_trace()
+
+# if __name__ == "__main__":
+#     file_share_app.run(port=3000)    
+
+with gr.Blocks(
+    analytics_enabled=False,
+    css=".gradio-container {background-color: #78BD91}",
+) as demo:
+    with gr.Column(elem_id="Column"):
+        input_format = gr.Radio(
+            choices=["microphone", "upload"], label="Choose your input format", elem_id="input_format"
+        )
+        input_audio = gr.Audio(
+            source="microphone",
+            type="filepath",
+            label="Input Audio",
+            interactive=True,
+            visible=False,
+            elem_id="input_audio"
+        )
+        input_format.change(
+            fn=change_audiobox, inputs=input_format, outputs=input_audio
+        )
+
+        speaker_option = gr.Radio(choices=spk_names, value="Male1", label="Choose your voice profile")
+        spk_icon = gr.Image(value="speaker_icons/male1.png",
+                            type="filepath",
+                            image_mode="RGB",
+                            source="upload",
+                            shape=[50, 50],
+                            interactive=True,
+                            visible=True)
+        speaker_option.change(
+            fn=show_icon, inputs=speaker_option, outputs=spk_icon
+        )
+
+    b2 = gr.Button("Convert")
+    
+    output_audio = gr.Audio(
+        source="upload", file="filepath", label="Converted Audio", interactive=False
+    )
+    
+    b2.click(
+        ASRTTS_clean,
+        inputs=[input_audio, speaker_option],
+        outputs=output_audio,
+        api_name="convert"
+    )
+    
+# download_file("wav/001_F1_spkembs.wav")
+
+demo.launch(share=False)

local/semi_streaming_ASR_TTS.py

@@ -0,0 +1,175 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import jiwer
+import pdb
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+# from time import process_time, time
+from pathlib import Path
+import time
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+
+# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [
+    str(x)
+    for x in sorted(
+        Path(
+            "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+        ).glob("**/*wav")
+    )
+]
+# audio_files = [str(x) for x in  sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+transcriber = pipeline(
+    "automatic-speech-recognition",
+    model="KevinGeng/PAL_John_128_train_dev_test_seed_1",
+)
+old_transcriber = pipeline(
+    "automatic-speech-recognition", "facebook/wav2vec2-base-960h"
+)
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+# @title English multi-speaker pretrained model { run: "auto" }
+lang = "English"
+tag = "kan-bayashi/libritts_xvector_vits"
+# vits needs no
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [
+    p
+    for p in glob.glob(
+        f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True
+    )
+    if "tr" in p
+][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+spks = list(xvectors.keys())
+
+male_spks = {
+    "M1": "2300_131720",
+    "M2": "1320_122612",
+    "M3": "1188_133604",
+    "M4": "61_70970",
+}
+female_spks = {"F1": "2961_961", "F2": "8463_287645", "F3": "121_121726"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+## 20230224 Mousa: No reference,
+def ASRold(audio_file):
+    reg_text = old_transcriber(audio_file)["text"]
+    return reg_text
+
+
+def ASRnew(audio_file, state=""):
+    # pdb.set_trace()
+    time.sleep(2)
+    reg_text = transcriber(audio_file)["text"]
+    state += reg_text + "\n"
+    return state, state
+
+def VAD(audio_file):
+    # pdb.set_trace()
+    reg_text = transcriber(audio_file)["text"]
+    return 1
+
+
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+
+input_audio = gr.Audio(
+    source="upload", type="filepath", label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+    source="upload", file="filepath", label="Synthesized Audio"
+)
+examples = [
+    ["./samples/001.wav", "M1", ""],
+    ["./samples/002.wav", "M2", ""],
+    ["./samples/003.wav", "F1", ""],
+    ["./samples/004.wav", "F2", ""],
+]
+
+def change_audiobox(choice):
+    if choice == "upload":
+        input_audio = gr.Audio.update(source="upload", visible=True)
+    elif choice == "microphone":
+        input_audio = gr.Audio.update(source="microphone", visible=True)
+    else:
+        input_audio = gr.Audio.update(visible=False)
+    return input_audio
+
+demo = gr.Interface(
+    fn=ASRnew, 
+    inputs=[
+        gr.Audio(source="microphone", type="filepath", streaming=True), 
+        "state" 
+    ],
+    outputs=[
+        "textbox",
+        "state"
+    ],
+    live=True)
+# ASRnew(["/home/kevingeng/Disk2/laronix/Laronix_ASR_TTS_VC/wav/20221228_video_good_normed_5/take1_001_norm.wav", "state"])
+# VAD("/home/kevingeng/Disk2/laronix/Laronix_ASR_TTS_VC/wav/20221228_video_good_normed_5/take1_001_norm.wav")
+demo.launch(share=False)

local/streaming_VAD.py

@@ -0,0 +1,74 @@
+import pyaudio
+import numpy as np
+import webrtcvad
+
+# Set up PyAudio
+FORMAT = pyaudio.paInt16
+CHANNELS = 1
+RATE = 48000
+CHUNK_SIZE = 960  # 20ms audio chunks
+# p = pyaudio.PyAudio()
+
+# wav = "/home/kevingeng/Disk2/laronix/Laronix_ASR_TTS_VC/wav/20221228_video_good_normed_5/take1_001_norm.wav"
+wav = "/home/kevingeng/Disk2/laronix/Laronix_ASR_TTS_VC/wav/VAD_test.wav"
+import wave
+wf = wave.open(wav, "rb")
+# import pdb
+# stream = p.open(format=p.get_format_from_width(wf.getsampwidth()),
+#                 channels=wf.getnchannels(),
+#                 rate=wf.getframerate(),
+#                 output=True)
+# pdb.set_trace()
+# Set up VAD
+
+def streaming_VAD(wf):
+    vad = webrtcvad.Vad()
+    vad.set_mode(2)  # Aggressive mode
+
+    # Start audio stream
+    # stream = p.open(format=FORMAT,
+    #                 channels=CHANNELS,
+    #                 rate=RATE,
+    #                 input=True,
+    #                 frames_per_buffer=CHUNK_SIZE)
+
+    # VAD constants
+    MIN_SILENCE_DURATION = 2000  # in ms
+    MAX_SILENCE_DURATION = 4000  # in ms
+    BUFFER_SIZE = MAX_SILENCE_DURATION // CHUNK_SIZE
+    BUFFER_THRESHOLD = int(BUFFER_SIZE * 0.5)
+
+    # Initialize VAD buffer
+    vad_buffer = []
+    VAD_indicator = []
+    VAD_frame_indicator = []
+    data = wf.readframes(CHUNK_SIZE)
+    # Loop through audio stream
+    while data:
+        # Read audio chunk from stream
+        # pdb.set_trace()
+        # audio_chunk = np.frombuffer(stream.read(CHUNK_SIZE), dtype=np.int16)
+        audio_chunk = np.frombuffer(data, dtype=np.int16)
+        # Detect voice activity
+        # is_speech = vad.is_speech(audio_chunk.tobytes(), RATE)
+        try:
+            is_speech = vad.is_speech(audio_chunk, RATE)
+        except:
+            is_speech = False
+        vad_buffer.append(is_speech)
+        
+        # If VAD buffer is full, check for silence and reset buffer
+        if len(vad_buffer) == BUFFER_SIZE:
+            # Check if buffer contains mostly silence
+            if vad_buffer.count(False) >= BUFFER_THRESHOLD:
+                # print("Slience")
+                # VAD_indicator.append(0)
+                # vad_buffer = []
+                return(False)
+            else:
+                # print("Voice detected!")
+                # VAD_indicator.append(1)
+                vad_buffer = vad_buffer[CHUNK_SIZE // BUFFER_SIZE:]
+                return(True)
+        data = wf.readframes(CHUNK_SIZE)
+

requirements.txt

@@ -28,7 +28,7 @@ fsspec==2022.2.0
 future==0.18.2
 google-auth==2.6.0
 google-auth-oauthlib==0.4.6
-gradio==3.2
+gradio==3.18
 grpcio==1.44.0
 h11==0.12.0
 hydra-core==1.0.7
@@ -108,3 +108,8 @@ jiwer
 # charset
 
 gradio
+
+flask
+
+# datasets
+datasets

requirements.txt.bak.bak

@@ -1,141 +0,0 @@
-aiofiles==23.1.0
-aiohttp==3.8.4
-aiosignal==1.3.1
-altair==4.2.2
-antlr4-python3-runtime==4.8
-anyio==3.6.2
-appdirs==1.4.4
-argcomplete==2.0.0
-async-timeout==4.0.2
-asynctest==0.13.0
-attrs==22.2.0
-audioread==3.0.0
-beautifulsoup4==4.11.2
-bitarray==2.7.2
-black==23.1.0
-brotlipy==0.7.0
-cchardet==2.1.7
-chardet==5.1.0
-charset-normalizer==3.0.1
-ci-sdr==0.0.2
-click==8.1.3
-colorama==0.4.6
-ConfigArgParse==1.5.3
-ctc-segmentation==1.7.4
-cycler==0.11.0
-Cython==0.29.33
-decorator==5.1.1
-Distance==0.1.3
-editdistance==0.6.2
-einops==0.6.0
-entrypoints==0.4
-espnet==202301
-espnet-model-zoo==0.1.7
-espnet-tts-frontend==0.0.3
-fairseq==0.12.2
-fast-bss-eval==0.1.3
-fastapi==0.91.0
-ffmpy==0.3.0
-filelock==3.9.0
-fonttools==4.38.0
-frozenlist==1.3.3
-fsspec==2023.1.0
-g2p-en==2.1.0
-gdown==4.6.3
-gradio==3.18.0
-h11==0.14.0
-h5py==3.8.0
-httpcore==0.16.3
-httpx==0.23.3
-huggingface-hub==0.12.0
-humanfriendly==10.0
-hydra-core==1.0.7
-importlib-metadata==4.13.0
-importlib-resources==5.10.2
-inflect==6.0.2
-jaconv==0.3.3
-jamo==0.4.1
-Jinja2==3.1.2
-jiwer==2.5.1
-joblib==1.2.0
-jsonschema==4.17.3
-kaldiio==2.17.2
-kiwisolver==1.4.4
-Levenshtein==0.20.2
-librosa==0.9.2
-linkify-it-py==1.0.3
-llvmlite==0.39.1
-lxml==4.9.2
-markdown-it-py==2.1.0
-MarkupSafe==2.1.2
-matplotlib==3.5.3
-mdit-py-plugins==0.3.3
-mdurl==0.1.2
-mkl-fft==1.3.1
-mkl-service==2.4.0
-multidict==6.0.4
-mypy-extensions==1.0.0
-nltk==3.8.1
-numba==0.56.4
-numpy==1.21.6
-omegaconf==2.0.6
-opt-einsum==3.3.0
-orjson==3.8.6
-packaging==23.0
-pandas==1.3.5
-parallel-wavegan==0.5.5
-pathspec==0.11.0
-Pillow==9.3.0
-pkgutil_resolve_name==1.3.10
-platformdirs==3.0.0
-pooch==1.6.0
-portalocker==2.7.0
-protobuf==3.20.1
-pycryptodome==3.17
-pydantic==1.10.4
-pydub==0.25.1
-pyparsing==3.0.9
-pypinyin==0.44.0
-pyrsistent==0.19.3
-python-dateutil==2.8.2
-python-multipart==0.0.5
-pytorch-wpe==0.0.1
-pytz==2022.7.1
-pyworld==0.3.2
-PyYAML==6.0
-rapidfuzz==2.13.7
-regex==2022.10.31
-requests==2.28.2
-resampy==0.4.2
-rfc3986==1.5.0
-sacrebleu==2.3.1
-scikit-learn==1.0.2
-scipy==1.7.3
-sentencepiece==0.1.97
-sniffio==1.3.0
-soundfile==0.11.0
-soupsieve==2.4
-starlette==0.24.0
-tabulate==0.9.0
-tensorboardX==2.6
-threadpoolctl==3.1.0
-tokenizers==0.13.2
-toml==0.10.2
-tomli==2.0.1
-toolz==0.12.0
-torch==1.12.1
-torch-complex==0.4.3
-torchaudio==0.12.1
-torchvision==0.13.1
-tqdm==4.64.1
-transformers==4.26.1
-typed-ast==1.5.4
-typeguard==2.13.3
-uc-micro-py==1.0.1
-Unidecode==1.3.6
-uvicorn==0.20.0
-websockets==10.4
-xmltodict==0.13.0
-yarl==1.8.2
-yq==3.1.0
-zipp==3.13.0