Add Files

.gitignore

@@ -0,0 +1,5 @@
+flagged
+wav
+samples
+wav
+wav.bak

local/ASR_compare.py

@@ -0,0 +1,214 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import jiwer
+import pdb
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from time import process_time, time
+from pathlib import Path
+
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+
+# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+sys.path.append("src")
+
+import gradio as gr
+
+# ASR part
+
+audio_files = [
+    str(x)
+    for x in sorted(
+        Path(
+            "/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video"
+        ).glob("**/*wav")
+    )
+]
+# audio_files = [str(x) for x in  sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+transcriber = pipeline(
+    "automatic-speech-recognition",
+    model="KevinGeng/PAL_John_128_train_dev_test_seed_1",
+)
+old_transcriber = pipeline(
+    "automatic-speech-recognition", "facebook/wav2vec2-base-960h"
+)
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+# @title English multi-speaker pretrained model { run: "auto" }
+lang = "English"
+tag = "kan-bayashi/libritts_xvector_vits"
+# vits needs no
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long"  # @param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+
+xvector_ark = [
+    p
+    for p in glob.glob(
+        f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True
+    )
+    if "tr" in p
+][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+spks = list(xvectors.keys())
+
+male_spks = {
+    "M1": "2300_131720",
+    "M2": "1320_122612",
+    "M3": "1188_133604",
+    "M4": "61_70970",
+}
+female_spks = {"F1": "2961_961", "F2": "8463_287645", "F3": "121_121726"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+
+
+## 20230224 Mousa: No reference,
+def ASRold(audio_file):
+    reg_text = old_transcriber(audio_file)["text"]
+    return reg_text
+
+
+def ASRnew(audio_file):
+    reg_text = transcriber(audio_file)["text"]
+    return reg_text
+
+
+# def ref_reg_callback(audio_file, spk_name, ref_text):
+#     reg_text = ref_text
+#     return audio_file, spk_name, reg_text
+
+reference_textbox = gr.Textbox(
+    value="",
+    placeholder="Input reference here",
+    label="Reference",
+)
+
+recognization_textbox = gr.Textbox(
+    value="",
+    placeholder="Output recognization here",
+    label="recognization_textbox",
+)
+
+speaker_option = gr.Radio(choices=spk_names, label="Speaker")
+# speaker_profiles = {
+#     "Male_1": "speaker_icons/male1.png",
+#     "Male_2": "speaker_icons/male2.png",
+#     "Female_1": "speaker_icons/female1.png",
+#     "Female_2": "speaker_icons/female2.png",
+# }
+
+# speaker_option = gr.Image(label="Choose your speaker profile",
+#                                  image_mode="RGB",
+#                                  options=speaker_profiles
+#                                  )
+
+input_audio = gr.Audio(
+    source="upload", type="filepath", label="Audio_to_Evaluate"
+)
+output_audio = gr.Audio(
+    source="upload", file="filepath", label="Synthesized Audio"
+)
+examples = [
+    ["./samples/001.wav", "M1", ""],
+    ["./samples/002.wav", "M2", ""],
+    ["./samples/003.wav", "F1", ""],
+    ["./samples/004.wav", "F2", ""],
+]
+
+
+def change_audiobox(choice):
+    if choice == "upload":
+        input_audio = gr.Audio.update(source="upload", visible=True)
+    elif choice == "microphone":
+        input_audio = gr.Audio.update(source="microphone", visible=True)
+    else:
+        input_audio = gr.Audio.update(visible=False)
+    return input_audio
+
+
+with gr.Blocks(
+    analytics_enabled=False,
+    css=".gradio-container {background-color: #78BD91}",
+) as demo:
+    with gr.Column():
+        input_format = gr.Radio(
+            choices=["upload", "microphone"], label="Choose your input format"
+        )
+        input_audio = gr.Audio(
+            source="upload",
+            type="filepath",
+            label="Input Audio",
+            interactive=True,
+            visible=False,
+        )
+        input_format.change(
+            fn=change_audiobox, inputs=input_format, outputs=input_audio
+        )
+
+    with gr.Row():
+        b1 = gr.Button("Conventional Speech Recognition Engine")
+        old_recognization_textbox = gr.Textbox(
+            value="",
+            placeholder="Recognition output",
+            label="Convertional",
+        )
+        b1.click(
+            ASRold, inputs=[input_audio], outputs=old_recognization_textbox
+        )
+
+    with gr.Row():
+        b2 = gr.Button("Laronix Speech Recognition Engine")
+        new_recognization_textbox = gr.Textbox(
+            value="",
+            placeholder="Recognition output",
+            label="Purposed",
+        )
+
+        b2.click(
+            ASRnew, inputs=[input_audio], outputs=new_recognization_textbox
+        )
+
+demo.launch(share=True)

local/app_batch.py

@@ -0,0 +1,94 @@
+"""
+TODO:
+    + [x] Load Configuration
+    + [ ] Checking
+    + [ ] Better saving directory
+"""
+import numpy as np
+from pathlib import Path
+import jiwer
+import pdb
+import torch.nn as nn
+import torch
+import torchaudio
+from transformers import pipeline
+from time import process_time, time
+from pathlib import Path
+# local import
+import sys
+from espnet2.bin.tts_inference import Text2Speech
+
+# pdb.set_trace()
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 
+
+sys.path.append("src")
+
+# ASR part
+
+audio_files = [str(x) for x in  sorted(Path("/home/kevingeng/Disk2/laronix/laronix_automos/data/20230103_video").glob("**/*wav"))]
+# audio_files = [str(x) for x in  sorted(Path("/mnt/Disk2/laronix/laronix_PAL_ASR_TTS/wav/20221228_video_good_normed_5").glob("**/*wav"))]
+# pdb.set_trace()
+# audio_files = [str(x) for x in  sorted(Path("./data/Patient_sil_trim_16k_normed_5_snr_40/Rainbow").glob("**/*wav"))]
+transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_train_dev_test_seed_1")
+# transcriber = pipeline("automatic-speech-recognition", model="KevinGeng/PAL_John_128_p326_300_train_dev_test_seed_1")
+# 【Female】kan-bayashi ljspeech parallel wavegan
+# tts_model = Text2Speech.from_pretrained("espnet/kan-bayashi_ljspeech_vits")
+# 【Male】fastspeech2-en-200_speaker-cv4, hifigan vocoder
+# pdb.set_trace()
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+
+#@title English multi-speaker pretrained model { run: "auto" }
+lang = 'English'
+# tag = 'kan-bayashi/vctk_multi_spk_vits' #@param ["kan-bayashi/vctk_gst_tacotron2", "kan-bayashi/vctk_gst_transformer", "kan-bayashi/vctk_xvector_tacotron2", "kan-bayashi/vctk_xvector_transformer", "kan-bayashi/vctk_xvector_conformer_fastspeech2", "kan-bayashi/vctk_gst+xvector_tacotron2", "kan-bayashi/vctk_gst+xvector_transformer", "kan-bayashi/vctk_gst+xvector_conformer_fastspeech2", "kan-bayashi/vctk_multi_spk_vits", "kan-bayashi/vctk_full_band_multi_spk_vits", "kan-bayashi/libritts_xvector_transformer", "kan-bayashi/libritts_xvector_conformer_fastspeech2", "kan-bayashi/libritts_gst+xvector_transformer", "kan-bayashi/libritts_gst+xvector_conformer_fastspeech2", "kan-bayashi/libritts_xvector_vits"] {type:"string"}
+tag = 'kan-bayashi/libritts_xvector_vits' 
+# vits needs no 
+vocoder_tag = "parallel_wavegan/vctk_parallel_wavegan.v1.long" #@param ["none", "parallel_wavegan/vctk_parallel_wavegan.v1.long", "parallel_wavegan/vctk_multi_band_melgan.v2", "parallel_wavegan/vctk_style_melgan.v1", "parallel_wavegan/vctk_hifigan.v1", "parallel_wavegan/libritts_parallel_wavegan.v1.long", "parallel_wavegan/libritts_multi_band_melgan.v2", "parallel_wavegan/libritts_hifigan.v1", "parallel_wavegan/libritts_style_melgan.v1"] {type:"string"}
+from espnet2.bin.tts_inference import Text2Speech
+from espnet2.utils.types import str_or_none
+
+text2speech = Text2Speech.from_pretrained(
+    model_tag=str_or_none(tag),
+    vocoder_tag=str_or_none(vocoder_tag),
+    device="cuda",
+    use_att_constraint=False,
+    backward_window=1,
+    forward_window=3,
+    speed_control_alpha=1.0,
+)
+
+
+import glob
+import os
+import numpy as np
+import kaldiio
+
+# Get model directory path
+from espnet_model_zoo.downloader import ModelDownloader
+d = ModelDownloader()
+model_dir = os.path.dirname(d.download_and_unpack(tag)["train_config"])
+
+# Speaker x-vector selection
+# pdb.set_trace()
+xvector_ark = [p for p in glob.glob(f"{model_dir}/../../dump/**/spk_xvector.ark", recursive=True) if "tr" in p][0]
+xvectors = {k: v for k, v in kaldiio.load_ark(xvector_ark)}
+# spks = list(xvectors.keys())
+
+male_spks = {"M1": "2300_131720", "M2": "1320_122612", "M3": "1188_133604", "M4": "61_70970"}
+female_spks = {"F1": "2961_961", "F2": "8463_287645", "F3": "121_121726"}
+spks = dict(male_spks, **female_spks)
+spk_names = sorted(spks.keys())
+# pdb.set_trace()
+selected_xvectors = [xvectors[x] for x in spks.values()]
+selected_xvectors_dict = dict(zip(spks.keys(), selected_xvectors))
+
+for audio_file in audio_files:
+    t_start = time()
+    text = transcriber(audio_file)['text']
+    speech, sr = torchaudio.load(audio_file) # reference speech
+    duration = len(speech)/sr
+    for spks,spembs in selected_xvectors_dict.items():
+        wav_tensor_spembs = text2speech(text=text, speech=speech, spembs=spembs)["wav"]
+        torchaudio.save("./wav/" + Path(audio_file).stem + "_" + spks +"_spkembs.wav", src=wav_tensor_spembs.unsqueeze(0).to("cpu"), sample_rate=22050)
+
+    # torchaudio.save("./wav/" + Path(audio_file).stem + "_" + spk + "_dur_t_text.wav", src=wav_tensor_duration_t_text.unsqueeze(0).to("cpu"), sample_rate=22050)

requirements.txt

@@ -0,0 +1,153 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+antlr4-python3-runtime==4.8
+anyio==3.6.2
+appdirs==1.4.4
+argcomplete==2.0.0
+async-timeout==4.0.2
+asynctest==0.13.0
+attrs==22.2.0
+audioread==3.0.0
+beautifulsoup4==4.11.2
+bitarray==2.7.2
+black==23.1.0
+brotlipy==0.7.0
+cchardet==2.1.7
+certifi @ file:///croot/certifi_1671487769961/work/certifi
+cffi @ file:///croot/cffi_1670423208954/work
+chardet==5.1.0
+charset-normalizer==3.0.1
+ci-sdr==0.0.2
+click==8.1.3
+colorama==0.4.6
+ConfigArgParse==1.5.3
+cryptography @ file:///croot/cryptography_1673298753778/work
+ctc-segmentation==1.7.4
+cycler==0.11.0
+Cython==0.29.33
+decorator==5.1.1
+Distance==0.1.3
+editdistance==0.6.2
+einops==0.6.0
+entrypoints==0.4
+espnet==202301
+espnet-model-zoo==0.1.7
+espnet-tts-frontend==0.0.3
+fairseq==0.12.2
+fast-bss-eval==0.1.3
+fastapi==0.91.0
+ffmpy==0.3.0
+filelock==3.9.0
+flit_core @ file:///opt/conda/conda-bld/flit-core_1644941570762/work/source/flit_core
+fonttools==4.38.0
+frozenlist==1.3.3
+fsspec==2023.1.0
+g2p-en==2.1.0
+gdown==4.6.3
+gradio==3.18.0
+h11==0.14.0
+h5py==3.8.0
+httpcore==0.16.3
+httpx==0.23.3
+huggingface-hub==0.12.0
+humanfriendly==10.0
+hydra-core==1.0.7
+idna @ file:///croot/idna_1666125576474/work
+importlib-metadata==4.13.0
+importlib-resources==5.10.2
+inflect==6.0.2
+jaconv==0.3.3
+jamo==0.4.1
+Jinja2==3.1.2
+jiwer==2.5.1
+joblib==1.2.0
+jsonschema==4.17.3
+kaldiio==2.17.2
+kiwisolver==1.4.4
+Levenshtein==0.20.2
+librosa==0.9.2
+linkify-it-py==1.0.3
+llvmlite==0.39.1
+lxml==4.9.2
+markdown-it-py==2.1.0
+MarkupSafe==2.1.2
+matplotlib==3.5.3
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mkl-fft==1.3.1
+mkl-random @ file:///tmp/build/80754af9/mkl_random_1626179032232/work
+mkl-service==2.4.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+nltk==3.8.1
+numba==0.56.4
+numpy==1.21.6
+omegaconf==2.0.6
+opt-einsum==3.3.0
+orjson==3.8.6
+packaging==23.0
+pandas==1.3.5
+parallel-wavegan==0.5.5
+pathspec==0.11.0
+Pillow==9.3.0
+pkgutil_resolve_name==1.3.10
+platformdirs==3.0.0
+pooch==1.6.0
+portalocker==2.7.0
+protobuf==3.20.1
+pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
+pycryptodome==3.17
+pydantic==1.10.4
+pydub==0.25.1
+pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work
+pyparsing==3.0.9
+pypinyin==0.44.0
+pyrsistent==0.19.3
+PySocks @ file:///tmp/build/80754af9/pysocks_1594394576006/work
+python-dateutil==2.8.2
+python-multipart==0.0.5
+pytorch-wpe==0.0.1
+pytz==2022.7.1
+pyworld==0.3.2
+PyYAML==6.0
+rapidfuzz==2.13.7
+regex==2022.10.31
+requests==2.28.2
+resampy==0.4.2
+rfc3986==1.5.0
+sacrebleu==2.3.1
+scikit-learn==1.0.2
+scipy==1.7.3
+sentencepiece==0.1.97
+six @ file:///tmp/build/80754af9/six_1644875935023/work
+sniffio==1.3.0
+soundfile==0.11.0
+soupsieve==2.4
+starlette==0.24.0
+tabulate==0.9.0
+tensorboardX==2.6
+threadpoolctl==3.1.0
+tokenizers==0.13.2
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.0
+torch==1.12.1
+torch-complex==0.4.3
+torchaudio==0.12.1
+torchvision==0.13.1
+tqdm==4.64.1
+transformers==4.26.1
+typed-ast==1.5.4
+typeguard==2.13.3
+typing_extensions @ file:///croot/typing_extensions_1669924550328/work
+uc-micro-py==1.0.1
+Unidecode==1.3.6
+urllib3 @ file:///croot/urllib3_1673575502006/work
+uvicorn==0.20.0
+websockets==10.4
+xmltodict==0.13.0
+yarl==1.8.2
+yq==3.1.0
+zipp==3.13.0