update

app.py

@@ -1,6 +1,123 @@
+import os
+import gradio as gr
+import whisper 
+import requests 
+import tempfile
+from neon_tts_plugin_coqui import CoquiTTS
 from datasets import load_dataset
 
 dataset = load_dataset("ysharma/short_jokes")
 
+# Language common in both the multilingual models - English, Chinese, Spanish, and French etc
+# So it would make sense to test the App on these four prominently
 
-print(dataset['train'][0])
+# Whisper: Speech-to-text
+model = whisper.load_model("base")
+model_med = whisper.load_model("medium")
+# Languages covered in Whisper - (exhaustive list) : 
+#"en": "english", "zh": "chinese", "de": "german", "es": "spanish", "ru": "russian", 
+#"ko": "korean", "fr": "french", "ja": "japanese", "pt": "portuguese", "tr": "turkish", 
+#"pl": "polish", "ca": "catalan", "nl": "dutch", "ar": "arabic", "sv": "swedish", 
+#"it": "italian", "id": "indonesian", "hi": "hindi", "fi": "finnish", "vi": "vietnamese", 
+#"iw": "hebrew", "uk": "ukrainian", "el": "greek", "ms": "malay", "cs": "czech", 
+#"ro": "romanian", "da": "danish", "hu": "hungarian", "ta": "tamil", "no": "norwegian", 
+#"th": "thai", "ur": "urdu", "hr": "croatian", "bg": "bulgarian", "lt": "lithuanian", 
+#"la": "latin", "mi": "maori", "ml": "malayalam", "cy": "welsh", "sk": "slovak", 
+#"te": "telugu", "fa": "persian", "lv": "latvian", "bn": "bengali", "sr": "serbian", 
+#"az": "azerbaijani", "sl": "slovenian", "kn": "kannada", "et": "estonian", 
+#"mk": "macedonian", "br": "breton", "eu": "basque", "is": "icelandic", "hy": "armenian", 
+#"ne": "nepali", "mn": "mongolian", "bs": "bosnian", "kk": "kazakh", "sq": "albanian", 
+#"sw": "swahili", "gl": "galician", "mr": "marathi", "pa": "punjabi", "si": "sinhala", 
+#"km": "khmer", "sn": "shona", "yo": "yoruba", "so": "somali", "af": "afrikaans", 
+#"oc": "occitan", "ka": "georgian", "be": "belarusian", "tg": "tajik", "sd": "sindhi", 
+#"gu": "gujarati", "am": "amharic", "yi": "yiddish", "lo": "lao", "uz": "uzbek", 
+#"fo": "faroese", "ht": "haitian creole", "ps": "pashto", "tk": "turkmen", "nn": "nynorsk", 
+#"mt": "maltese", "sa": "sanskrit", "lb": "luxembourgish", "my": "myanmar", "bo": "tibetan", 
+#"tl": "tagalog", "mg": "malagasy", "as": "assamese", "tt": "tatar", "haw": "hawaiian", 
+#"ln": "lingala", "ha": "hausa", "ba": "bashkir", "jw": "javanese", "su": "sundanese",
+
+
+# Text-to-Speech
+LANGUAGES = list(CoquiTTS.langs.keys())
+coquiTTS = CoquiTTS()
+print(f"Languages for Coqui are: {LANGUAGES}")
+#Languages for Coqui are: ['en', 'es', 'fr', 'de', 'pl', 'uk', 'ro', 'hu', 'el', 'bg', 'nl', 'fi', 'sl', 'lv', 'ga']
+# en - English, es - Spanish, fr -  French, de - German, pl - Polish
+# uk - Ukrainian, ro - Romanian, hu - Hungarian, el - Greek, bg - Bulgarian,
+# nl - dutch, fi - finnish, sl - slovenian, lv - latvian, ga - ??  
+
+
+# Driver function
+def driver_fun(audio) : 
+  transcribe, translation, lang = whisper_stt(audio)
+  #text1 = model.transcribe(audio)["text"]
+  
+  #For now only taking in English text for Bloom prompting as inference model is not high spec
+  #text_generated = lang_model_response(transcribe, lang)
+  #text_generated_en = lang_model_response(translation, 'en')
+  
+  if lang in ['es', 'fr']:
+    speech = tts(transcribe, lang)
+  else:
+    speech = tts(translation, 'en') #'en')
+  return transcribe, translation, speech
+
+
+# Whisper - speech-to-text
+def whisper_stt(audio):
+  print("Inside Whisper TTS")
+  # load audio and pad/trim it to fit 30 seconds
+  audio = whisper.load_audio(audio)
+  audio = whisper.pad_or_trim(audio)
+  
+  # make log-Mel spectrogram and move to the same device as the model
+  mel = whisper.log_mel_spectrogram(audio).to(model.device)
+  
+  # detect the spoken language
+  _, probs = model.detect_language(mel)
+  lang = max(probs, key=probs.get)
+  print(f"Detected language: {max(probs, key=probs.get)}")
+  
+  # decode the audio
+  options_transc = whisper.DecodingOptions(fp16 = False, language=lang, task='transcribe') #lang
+  options_transl = whisper.DecodingOptions(fp16 = False, language='en', task='translate') #lang
+  result_transc = whisper.decode(model_med, mel, options_transc)
+  result_transl = whisper.decode(model_med, mel, options_transl)
+  
+  # print the recognized text
+  print(f"transcript is : {result_transc.text}")
+  print(f"translation is : {result_transl.text}")
+
+  return result_transc.text, result_transl.text, lang
+
+
+# Coqui - Text-to-Speech
+def tts(text, language):
+  print(f"Inside tts - language is : {language}")
+  coqui_langs = ['en' ,'es' ,'fr' ,'de' ,'pl' ,'uk' ,'ro' ,'hu' ,'bg' ,'nl' ,'fi' ,'sl' ,'lv' ,'ga']
+  if language not in coqui_langs:
+    language = 'en'
+  with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
+      coquiTTS.get_tts(text, fp, speaker = {"language" : language})
+      return fp.name
+
+demo = gr.Blocks()
+with demo:
+  gr.Markdown("<h1><center>Multilingual AI Assistant - Voice to Joke</center></h1>")
+  gr.Markdown(
+        """Model pipeline consisting of - <br>- [**Whisper**](https://github.com/openai/whisper) for Speech-to-text, <br>- [**CoquiTTS**](https://huggingface.co/coqui)  for Text-To-Speech. <br>- Front end is built using [**Gradio Block API**](https://gradio.app/docs/#blocks).<br>Both CoquiTTS and Whisper are Multilingual, there are several overlapping languages between them. Hence it would be suggested to test this ML-App using these two languages to get the best results</u>.<br>If you want to reuse the App, simply click on the small cross button in the top right corner of your voice record panel, and then press record again!
+        """)
+  with gr.Row():
+    with gr.Column(): 
+      in_audio = gr.Audio(source="microphone",  type="filepath", label='Record your voice command here in English, Spanish or French for best results-')  #type='filepath'
+      b1 = gr.Button("AI response pipeline (Whisper - Bloom - Coqui pipeline)")
+      out_transcript = gr.Textbox(label= 'English/Spanish/French Transcript of your Audio using OpenAI Whisper')
+      out_translation_en = gr.Textbox(label= 'English Translation of audio using OpenAI Whisper')
+    with gr.Column():
+      out_audio = gr.Audio(label='AI response in Audio form in your language - This will be either in Spanish, or in French or in English for all other languages -')  
+      out_generated_text = gr.Textbox(label= 'AI response to your query in your preferred language using Bloom! ')
+      out_generated_text_en = gr.Textbox(label= 'AI response to your query in English using Bloom! ')
+    
+      b1.click(driver_fun,inputs=[in_audio], outputs=[out_transcript, out_translation_en, out_generated_text,out_generated_text_en, out_audio]) 
+    
+demo.launch(enable_queue=True, debug=True)