init

app.py

@@ -0,0 +1,78 @@
+import gradio as gr
+from transformers import pipeline
+import time
+from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
+from fairseq.models.text_to_speech.hub_interface import TTSHubInterface
+import torch
+from fairseq.utils import move_to_cuda
+
+"""
+Notes:
+pip install sentencepiece
+pip install phonemizer
+
+install ffmpeg and add to path
+
+Must install e-speak to path 
+
+solution
+1.Download and install the Windows version of espeak: http://espeak.sourceforge.net/download.html
+
+2. set PATH=%PATH%;"C:\Program Files (x86)\eSpeak\command_line"_
+
+3. Install .msi from https://github.com/espeak-ng/espeak-ng/releases
+
+4.Enter environment variable
+
+    1.PHONEMIZER_ESPEAK_LIBRARY="c:\Program Files\eSpeak NG\libespeak-ng.dll"
+    2.PHONEMIZER_ESPEAK_PATH =“c:\Program Files\eSpeak NG”
+
+and Restart your Computer. Run the same command again from the Command Prompt (cmd.exe):
+
+
+"""
+asr = pipeline("automatic-speech-recognition",model="facebook/wav2vec2-base-960h" )
+translation_pipeline = pipeline('translation_en_to_es',model = "Helsinki-NLP/opus-mt-en-es" ) #This model version is built for en- to -fr , less mistakes
+
+models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
+    "facebook/tts_transformer-es-css10",
+    arg_overrides={"vocoder": "hifigan", "fp16": False}
+)
+
+model = models[0]
+model = model.to(torch.device("cuda:0"))
+
+TTSHubInterface.update_cfg_with_data_cfg(cfg, task.data_cfg)
+generator = task.build_generator(models, cfg)
+
+
+def transcribe_translate(audio):
+    time.sleep(3)
+    text_en = asr(audio)["text"]
+    
+    text_fr = translation_pipeline(text_en.lower()) # for some reason all audio converted to all caps and it translates differently???
+    text_fr = text_fr[0]['translation_text']       # good evening = bonsoir  but GOOD EVENING = BONNES SÉANCES . WEIRD
+
+    sample = TTSHubInterface.get_model_input(task, text_fr)
+    sample = move_to_cuda(sample) if torch.cuda.is_available() else sample
+
+    wav, rate = TTSHubInterface.get_prediction(task, model, generator, sample)
+    wav = wav.to('cpu')
+    wav = wav.numpy()
+    print(wav.dtype)
+
+    return text_en,text_fr , (rate,wav)
+
+
+gr.Interface(
+    fn=transcribe_translate, 
+    inputs=[
+        gr.Audio(source="microphone", type="filepath")
+    ],
+    outputs=[
+        gr.Textbox(label= "English Transcription"),
+        gr.Textbox(label= "Spanish Translation"),
+        gr.Audio(label = "Spanish Audio")
+    ],
+    live=True).launch(share=True)
+

requirements.txt

@@ -0,0 +1,5 @@
+transformers
+fairseq
+torch
+sentencepiece
+phonemizer