update

app.py

@@ -5,32 +5,70 @@ import requests
 import tempfile
 from neon_tts_plugin_coqui import CoquiTTS
 
+# Language common in all three multilingual models - English, Chinese, Spanish, and French
+# So it would make sense to test the App on these four prominently
+
 # Whisper: Speech-to-text
 model = whisper.load_model("base")
+model_medium = whisper.load_model("medium")
+# Languages covered in Whisper - (exhaustive list) :
+#"en": "english", "zh": "chinese", "de": "german", "es": "spanish", "ru": "russian", 
+#"ko": "korean", "fr": "french", "ja": "japanese", "pt": "portuguese", "tr": "turkish", 
+#"pl": "polish", "ca": "catalan", "nl": "dutch", "ar": "arabic", "sv": "swedish", 
+#"it": "italian", "id": "indonesian", "hi": "hindi", "fi": "finnish", "vi": "vietnamese", 
+#"iw": "hebrew", "uk": "ukrainian", "el": "greek", "ms": "malay", "cs": "czech", 
+#"ro": "romanian", "da": "danish", "hu": "hungarian", "ta": "tamil", "no": "norwegian", 
+#"th": "thai", "ur": "urdu", "hr": "croatian", "bg": "bulgarian", "lt": "lithuanian", 
+#"la": "latin", "mi": "maori", "ml": "malayalam", "cy": "welsh", "sk": "slovak", 
+#"te": "telugu", "fa": "persian", "lv": "latvian", "bn": "bengali", "sr": "serbian", 
+#"az": "azerbaijani", "sl": "slovenian", "kn": "kannada", "et": "estonian", 
+#"mk": "macedonian", "br": "breton", "eu": "basque", "is": "icelandic", "hy": "armenian", 
+#"ne": "nepali", "mn": "mongolian", "bs": "bosnian", "kk": "kazakh", "sq": "albanian", 
+#"sw": "swahili", "gl": "galician", "mr": "marathi", "pa": "punjabi", "si": "sinhala", 
+#"km": "khmer", "sn": "shona", "yo": "yoruba", "so": "somali", "af": "afrikaans", 
+#"oc": "occitan", "ka": "georgian", "be": "belarusian", "tg": "tajik", "sd": "sindhi", 
+#"gu": "gujarati", "am": "amharic", "yi": "yiddish", "lo": "lao", "uz": "uzbek", 
+#"fo": "faroese", "ht": "haitian creole", "ps": "pashto", "tk": "turkmen", "nn": "nynorsk", 
+#"mt": "maltese", "sa": "sanskrit", "lb": "luxembourgish", "my": "myanmar", "bo": "tibetan", 
+#"tl": "tagalog", "mg": "malagasy", "as": "assamese", "tt": "tatar", "haw": "hawaiian", 
+#"ln": "lingala", "ha": "hausa", "ba": "bashkir", "jw": "javanese", "su": "sundanese",
+
 
 # LLM : Bloom as inference
 API_URL = "https://api-inference.huggingface.co/models/bigscience/bloom"
 HF_TOKEN = os.environ["HF_TOKEN"]
 headers = {"Authorization": f"Bearer {HF_TOKEN}"}
-#Language covered in Bloom : en, fr, esp, arb, hn, portu, Indonesian, Vietnamese, Chinese, tamil, telugu, bengali
+# Main Languages covered in Bloom are (not exhaustive list): 
+# English, Chinese, French, Spanish, Portuguese, Arabic, Hindi, Vietnamese, Indonesian, Bengali, Tamil, Telugu
+
 
 # Text-to-Speech
 LANGUAGES = list(CoquiTTS.langs.keys())
+coquiTTS = CoquiTTS()
 print(f"Languages for Coqui are: {LANGUAGES}")
 #Languages for Coqui are: ['en', 'es', 'fr', 'de', 'pl', 'uk', 'ro', 'hu', 'el', 'bg', 'nl', 'fi', 'sl', 'lv', 'ga']
-coquiTTS = CoquiTTS()
+# en - Engish, es - Spanish, fr -  French, de - German, pl - Polish
+# uk - Ukrainian, ro - Romanian, hu - Hungarian, el - Greek, bg - Bulgarian,
+# nl - dutch, fi - finnish, sl - slovenian, lv - latvian, ga - ??  
 
 
 # Driver function
 def driver_fun(audio) : 
-  text1, lang = whisper_stt(audio)
+  transcribe, translation, lang = whisper_stt(audio)
   #text1 = model.transcribe(audio)["text"]
-  text2 = lang_model_response(text1, lang)
-  speech = tts(text2, lang) #'en')
-  return text1, text2, speech
+  
+  #For now only taking in English text for Bloom prompting as inference model is not high spec
+  text_generated = lang_model_response(transcribe, lang)
+  text_generate_en = lang_model_response(translation, 'en')
+  
+  if lang in ['es', 'fr']:
+    speech = tts(text_generated, lang)
+  else:
+    speech = tts(text_generated_en, 'en') #'en')
+  return transcribe, translation, text_generate, text_generate_en, speech
 
 
-# Whisper - speeech-to-text
+# Whisper - speech-to-text
 def whisper_stt(audio):
   print("Inside Whisper TTS")
   # load audio and pad/trim it to fit 30 seconds
@@ -46,17 +84,28 @@ def whisper_stt(audio):
   print(f"Detected language: {max(probs, key=probs.get)}")
   
   # decode the audio
-  options = whisper.DecodingOptions(fp16 = False, language=lang)
-  result = whisper.decode(model, mel, options)
+  options_transc = whisper.DecodingOptions(fp16 = False, language=lang, task='transcribe') #lang
+  options_transl = whisper.DecodingOptions(fp16 = False, language='en', task='translate') #lang
+  result_transc = whisper.decode(model_med, mel, options_transc)
+  result_transl = whisper.decode(model_med, mel, options_transl)
+  
+  # print the recognized text
+  print(f"transcript is : {result_transc.text}")
+  print(f"translation is : {result_transl.text}")
+  
+  # decode the audio
+  #options = whisper.DecodingOptions(fp16 = False, language='en') #lang
+  #result = whisper.decode(model, mel, options)
   
   # print the recognized text
-  print(f"transcript is : {result.text}")
-  return result.text, lang
+  # print(f"transcript is : {result.text}")
+  # return result.text, lang
+  return result_transc.text, result_transl.text, lang
 
 
 # LLM - Bloom Response  
 def lang_model_response(prompt, language): 
-  print(f"*****Inside lang_model_response - Prompt is :{prompt}")
+  print(f"Inside lang_model_response - Prompt is :{prompt}")
   p = """Question: How are you doing today?
   Answer: I am doing good, thanks.
   Question: """
@@ -65,8 +114,10 @@ def lang_model_response(prompt, language):
     Answer: Sure, I am here for you.
     Question: """
   
-  prompt = p + prompt + "\n" + "Answer: "
-  
+  if language = 'en':
+    prompt = p + prompt + "\n" + "Answer: "
+  #else:
+    
   json_ = {"inputs": prompt,
             "parameters":
             {
@@ -85,7 +136,10 @@ def lang_model_response(prompt, language):
   output = response.json()
   output_tmp = output[0]['generated_text']
   print(f"Bloom API Response is : {output_tmp}")
-  solution = output_tmp.split("Answer: ")[2].split("\n")[0]
+  if language == 'en':
+    solution = output_tmp.split("Answer: ")[2].split("\n")[0]
+  else:
+    output_tmp.split(".")[1]
   print(f"Final Bloom Response after splits is: {solution}")
   return solution
 
@@ -99,7 +153,11 @@ def tts(text, language):
       coquiTTS.get_tts(text, fp, speaker = {"language" : language})
       return fp.name
 
+#demo = gr.Blocks()
+#with demo:
+#  gr.Markdown("<h1><center>Testing</center></h1>")
   
+    
 gr.Interface(
     title = 'Testing Whisper', 
     fn=driver_fun, 
@@ -108,6 +166,6 @@ gr.Interface(
        # "state"
     ],
     outputs=[
-        "textbox",  "textbox", "audio",
+        "textbox",  "textbox", "textbox", "textbox", "audio",
     ],
     live=True).launch()