ysharma's picture
ysharma HF staff
update
c9f7d28
raw
history blame
8.19 kB
import os
import gradio as gr
import whisper
import requests
import tempfile
from neon_tts_plugin_coqui import CoquiTTS
# Language common in all three multilingual models - English, Chinese, Spanish, and French
# So it would make sense to test the App on these four prominently
# Whisper: Speech-to-text
model = whisper.load_model("base")
model_med = whisper.load_model("medium")
# Languages covered in Whisper - (exhaustive list) :
#"en": "english", "zh": "chinese", "de": "german", "es": "spanish", "ru": "russian",
#"ko": "korean", "fr": "french", "ja": "japanese", "pt": "portuguese", "tr": "turkish",
#"pl": "polish", "ca": "catalan", "nl": "dutch", "ar": "arabic", "sv": "swedish",
#"it": "italian", "id": "indonesian", "hi": "hindi", "fi": "finnish", "vi": "vietnamese",
#"iw": "hebrew", "uk": "ukrainian", "el": "greek", "ms": "malay", "cs": "czech",
#"ro": "romanian", "da": "danish", "hu": "hungarian", "ta": "tamil", "no": "norwegian",
#"th": "thai", "ur": "urdu", "hr": "croatian", "bg": "bulgarian", "lt": "lithuanian",
#"la": "latin", "mi": "maori", "ml": "malayalam", "cy": "welsh", "sk": "slovak",
#"te": "telugu", "fa": "persian", "lv": "latvian", "bn": "bengali", "sr": "serbian",
#"az": "azerbaijani", "sl": "slovenian", "kn": "kannada", "et": "estonian",
#"mk": "macedonian", "br": "breton", "eu": "basque", "is": "icelandic", "hy": "armenian",
#"ne": "nepali", "mn": "mongolian", "bs": "bosnian", "kk": "kazakh", "sq": "albanian",
#"sw": "swahili", "gl": "galician", "mr": "marathi", "pa": "punjabi", "si": "sinhala",
#"km": "khmer", "sn": "shona", "yo": "yoruba", "so": "somali", "af": "afrikaans",
#"oc": "occitan", "ka": "georgian", "be": "belarusian", "tg": "tajik", "sd": "sindhi",
#"gu": "gujarati", "am": "amharic", "yi": "yiddish", "lo": "lao", "uz": "uzbek",
#"fo": "faroese", "ht": "haitian creole", "ps": "pashto", "tk": "turkmen", "nn": "nynorsk",
#"mt": "maltese", "sa": "sanskrit", "lb": "luxembourgish", "my": "myanmar", "bo": "tibetan",
#"tl": "tagalog", "mg": "malagasy", "as": "assamese", "tt": "tatar", "haw": "hawaiian",
#"ln": "lingala", "ha": "hausa", "ba": "bashkir", "jw": "javanese", "su": "sundanese",
# LLM : Bloom as inference
API_URL = "https://api-inference.huggingface.co/models/bigscience/bloom"
HF_TOKEN = os.environ["HF_TOKEN"]
headers = {"Authorization": f"Bearer {HF_TOKEN}"}
# Main Languages covered in Bloom are (not exhaustive list):
# English, Chinese, French, Spanish, Portuguese, Arabic, Hindi, Vietnamese, Indonesian, Bengali, Tamil, Telugu
# Text-to-Speech
LANGUAGES = list(CoquiTTS.langs.keys())
coquiTTS = CoquiTTS()
print(f"Languages for Coqui are: {LANGUAGES}")
#Languages for Coqui are: ['en', 'es', 'fr', 'de', 'pl', 'uk', 'ro', 'hu', 'el', 'bg', 'nl', 'fi', 'sl', 'lv', 'ga']
# en - Engish, es - Spanish, fr - French, de - German, pl - Polish
# uk - Ukrainian, ro - Romanian, hu - Hungarian, el - Greek, bg - Bulgarian,
# nl - dutch, fi - finnish, sl - slovenian, lv - latvian, ga - ??
# Driver function
def driver_fun(audio) :
transcribe, translation, lang = whisper_stt(audio)
#text1 = model.transcribe(audio)["text"]
#For now only taking in English text for Bloom prompting as inference model is not high spec
text_generated = lang_model_response(transcribe, lang)
text_generated_en = lang_model_response(translation, 'en')
if lang in ['es', 'fr']:
speech = tts(text_generated, lang)
else:
speech = tts(text_generated_en, 'en') #'en')
return transcribe, translation, text_generated, text_generated_en, speech
# Whisper - speech-to-text
def whisper_stt(audio):
print("Inside Whisper TTS")
# load audio and pad/trim it to fit 30 seconds
audio = whisper.load_audio(audio)
audio = whisper.pad_or_trim(audio)
# make log-Mel spectrogram and move to the same device as the model
mel = whisper.log_mel_spectrogram(audio).to(model.device)
# detect the spoken language
_, probs = model.detect_language(mel)
lang = max(probs, key=probs.get)
print(f"Detected language: {max(probs, key=probs.get)}")
# decode the audio
options_transc = whisper.DecodingOptions(fp16 = False, language=lang, task='transcribe') #lang
options_transl = whisper.DecodingOptions(fp16 = False, language='en', task='translate') #lang
result_transc = whisper.decode(model_med, mel, options_transc)
result_transl = whisper.decode(model_med, mel, options_transl)
# print the recognized text
print(f"transcript is : {result_transc.text}")
print(f"translation is : {result_transl.text}")
return result_transc.text, result_transl.text, lang
# LLM - Bloom Response
def lang_model_response(prompt, language):
print(f"Inside lang_model_response - Prompt is :{prompt}")
p_en = """Question: How are you doing today?
Answer: I am doing good, thanks.
Question: """
p_es = """Pregunta: Cómo estás hoy?
Responder: Estoy bien, gracias.
Pregunta: """
p_fr = """Question: Comment vas-tu aujourd'hui?
Réponse: Je vais bien, merci.
Question: """
if len(prompt) == 0:
prompt = """Question: Can you help me please?
Answer: Sure, I am here for you.
Question: """
if language == 'en':
prompt = p_en + prompt + "\n" + "Answer: "
elif language == 'es':
prompt = p_es + prompt + "\n" + "Responder: "
elif language == 'fr':
prompt = p_fr + prompt + "\n" + "Réponse: "
json_ = {"inputs": prompt,
"parameters":
{
"top_p": 0.90, #0.90 default
"max_new_tokens": 64,
"temperature": 1.1, #1.1 default
"return_full_text": False,
"do_sample": True,
},
"options":
{"use_cache": True,
"wait_for_model": True,
},}
response = requests.post(API_URL, headers=headers, json=json_)
#print(f"Response is : {response}")
output = response.json()
output_tmp = output[0]['generated_text']
print(f"Bloom API Response is : {output_tmp}")
#if language == 'en':
solution = output_tmp.split("Answer: ")[2].split("\n")[0]
#else:
# solution = output_tmp.split(".")[1]
print(f"Final Bloom Response after splits is: {solution}")
return solution
# Coqui - Text-to-Speech
def tts(text, language):
print(f"Inside tts - language is : {language}")
coqui_langs = ['en' ,'es' ,'fr' ,'de' ,'pl' ,'uk' ,'ro' ,'hu' ,'bg' ,'nl' ,'fi' ,'sl' ,'lv' ,'ga']
if language not in coqui_langs:
language = 'en'
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
coquiTTS.get_tts(text, fp, speaker = {"language" : language})
return fp.name
demo = gr.Blocks()
with demo:
gr.Markdown("<h1><center>Talk to Your Multilingual AI Assistant</center></h1>")
gr.Markdown(
"""Model pipeline consisting of - <br>- **Whisper** for Speech-to-text, <br>- **Bloom** for Text-generation, and <br>- **CoquiTTS** for Text-To-Speech. <br><br> Front end is built using **Gradio Block API**.
""")
with gr.Row():
with gr.Column():
in_audio = gr.Audio(source="microphone", type="filepath", label='Record your voice here') #type='filepath'
b1 = gr.Button("AI response (Whisper - Bloom - Coqui pipeline)")
out_transcript = gr.Textbox(label= 'As is Transcript using OpenAI Whisper')
out_translation_en = gr.Textbox(label= 'English Translation of audio using OpenAI Whisper')
with gr.Column():
out_audio = gr.Audio(label='AI response in Audio form in your preferred language')
out_generated_text = gr.Textbox(label= 'AI response to your query in your preferred language using Bloom! ')
out_generated_text_en = gr.Textbox(label= 'AI response to your query in English using Bloom! ')
b1.click(driver_fun,inputs=[in_audio], outputs=[out_transcript, out_translation_en, out_generated_text,out_generated_text_en, out_audio])
demo.launch(enable_queue=True, debug=True)
#gr.Interface(
# title = 'Testing Whisper',
# fn=driver_fun,
# inputs=[
# gr.Audio(source="microphone", type="filepath"), #streaming = True,
# # "state"
# ],
# outputs=[
# "textbox", "textbox", "textbox", "textbox", "audio",
# ],
# live=True).launch()