ysharma HF staff commited on
Commit
e50e0dc
1 Parent(s): 0cd60b8

create app.py

Browse files
Files changed (1) hide show
  1. app.py +171 -0
app.py ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import gradio as gr
3
+ import whisper
4
+ import requests
5
+ import tempfile
6
+ from neon_tts_plugin_coqui import CoquiTTS
7
+
8
+ # Language common in all three multilingual models - English, Chinese, Spanish, and French
9
+ # So it would make sense to test the App on these four prominently
10
+
11
+ # Whisper: Speech-to-text
12
+ model = whisper.load_model("base")
13
+ model_med = whisper.load_model("medium")
14
+ # Languages covered in Whisper - (exhaustive list) :
15
+ #"en": "english", "zh": "chinese", "de": "german", "es": "spanish", "ru": "russian",
16
+ #"ko": "korean", "fr": "french", "ja": "japanese", "pt": "portuguese", "tr": "turkish",
17
+ #"pl": "polish", "ca": "catalan", "nl": "dutch", "ar": "arabic", "sv": "swedish",
18
+ #"it": "italian", "id": "indonesian", "hi": "hindi", "fi": "finnish", "vi": "vietnamese",
19
+ #"iw": "hebrew", "uk": "ukrainian", "el": "greek", "ms": "malay", "cs": "czech",
20
+ #"ro": "romanian", "da": "danish", "hu": "hungarian", "ta": "tamil", "no": "norwegian",
21
+ #"th": "thai", "ur": "urdu", "hr": "croatian", "bg": "bulgarian", "lt": "lithuanian",
22
+ #"la": "latin", "mi": "maori", "ml": "malayalam", "cy": "welsh", "sk": "slovak",
23
+ #"te": "telugu", "fa": "persian", "lv": "latvian", "bn": "bengali", "sr": "serbian",
24
+ #"az": "azerbaijani", "sl": "slovenian", "kn": "kannada", "et": "estonian",
25
+ #"mk": "macedonian", "br": "breton", "eu": "basque", "is": "icelandic", "hy": "armenian",
26
+ #"ne": "nepali", "mn": "mongolian", "bs": "bosnian", "kk": "kazakh", "sq": "albanian",
27
+ #"sw": "swahili", "gl": "galician", "mr": "marathi", "pa": "punjabi", "si": "sinhala",
28
+ #"km": "khmer", "sn": "shona", "yo": "yoruba", "so": "somali", "af": "afrikaans",
29
+ #"oc": "occitan", "ka": "georgian", "be": "belarusian", "tg": "tajik", "sd": "sindhi",
30
+ #"gu": "gujarati", "am": "amharic", "yi": "yiddish", "lo": "lao", "uz": "uzbek",
31
+ #"fo": "faroese", "ht": "haitian creole", "ps": "pashto", "tk": "turkmen", "nn": "nynorsk",
32
+ #"mt": "maltese", "sa": "sanskrit", "lb": "luxembourgish", "my": "myanmar", "bo": "tibetan",
33
+ #"tl": "tagalog", "mg": "malagasy", "as": "assamese", "tt": "tatar", "haw": "hawaiian",
34
+ #"ln": "lingala", "ha": "hausa", "ba": "bashkir", "jw": "javanese", "su": "sundanese",
35
+
36
+
37
+ # LLM : Bloom as inference
38
+ API_URL = "https://api-inference.huggingface.co/models/bigscience/bloom"
39
+ HF_TOKEN = os.environ["HF_TOKEN"]
40
+ headers = {"Authorization": f"Bearer {HF_TOKEN}"}
41
+ # Main Languages covered in Bloom are (not exhaustive list):
42
+ # English, Chinese, French, Spanish, Portuguese, Arabic, Hindi, Vietnamese, Indonesian, Bengali, Tamil, Telugu
43
+
44
+
45
+ # Text-to-Speech
46
+ LANGUAGES = list(CoquiTTS.langs.keys())
47
+ coquiTTS = CoquiTTS()
48
+ print(f"Languages for Coqui are: {LANGUAGES}")
49
+ #Languages for Coqui are: ['en', 'es', 'fr', 'de', 'pl', 'uk', 'ro', 'hu', 'el', 'bg', 'nl', 'fi', 'sl', 'lv', 'ga']
50
+ # en - Engish, es - Spanish, fr - French, de - German, pl - Polish
51
+ # uk - Ukrainian, ro - Romanian, hu - Hungarian, el - Greek, bg - Bulgarian,
52
+ # nl - dutch, fi - finnish, sl - slovenian, lv - latvian, ga - ??
53
+
54
+
55
+ # Driver function
56
+ def driver_fun(audio) :
57
+ transcribe, translation, lang = whisper_stt(audio)
58
+ #text1 = model.transcribe(audio)["text"]
59
+
60
+ #For now only taking in English text for Bloom prompting as inference model is not high spec
61
+ text_generated = lang_model_response(transcribe, lang)
62
+ text_generate_en = lang_model_response(translation, 'en')
63
+
64
+ if lang in ['es', 'fr']:
65
+ speech = tts(text_generated, lang)
66
+ else:
67
+ speech = tts(text_generated_en, 'en') #'en')
68
+ return transcribe, translation, text_generate, text_generate_en, speech
69
+
70
+
71
+ # Whisper - speech-to-text
72
+ def whisper_stt(audio):
73
+ print("Inside Whisper TTS")
74
+ # load audio and pad/trim it to fit 30 seconds
75
+ audio = whisper.load_audio(audio)
76
+ audio = whisper.pad_or_trim(audio)
77
+
78
+ # make log-Mel spectrogram and move to the same device as the model
79
+ mel = whisper.log_mel_spectrogram(audio).to(model.device)
80
+
81
+ # detect the spoken language
82
+ _, probs = model.detect_language(mel)
83
+ lang = max(probs, key=probs.get)
84
+ print(f"Detected language: {max(probs, key=probs.get)}")
85
+
86
+ # decode the audio
87
+ options_transc = whisper.DecodingOptions(fp16 = False, language=lang, task='transcribe') #lang
88
+ options_transl = whisper.DecodingOptions(fp16 = False, language='en', task='translate') #lang
89
+ result_transc = whisper.decode(model_med, mel, options_transc)
90
+ result_transl = whisper.decode(model_med, mel, options_transl)
91
+
92
+ # print the recognized text
93
+ print(f"transcript is : {result_transc.text}")
94
+ print(f"translation is : {result_transl.text}")
95
+
96
+ # decode the audio
97
+ #options = whisper.DecodingOptions(fp16 = False, language='en') #lang
98
+ #result = whisper.decode(model, mel, options)
99
+
100
+ # print the recognized text
101
+ # print(f"transcript is : {result.text}")
102
+ # return result.text, lang
103
+ return result_transc.text, result_transl.text, lang
104
+
105
+
106
+ # LLM - Bloom Response
107
+ def lang_model_response(prompt, language):
108
+ print(f"Inside lang_model_response - Prompt is :{prompt}")
109
+ p = """Question: How are you doing today?
110
+ Answer: I am doing good, thanks.
111
+ Question: """
112
+ if len(prompt) == 0:
113
+ prompt = """Question: Can you help me please?
114
+ Answer: Sure, I am here for you.
115
+ Question: """
116
+
117
+ if language == 'en':
118
+ prompt = p + prompt + "\n" + "Answer: "
119
+ #else:
120
+
121
+ json_ = {"inputs": prompt,
122
+ "parameters":
123
+ {
124
+ "top_p": 0.90, #0.90 default
125
+ "max_new_tokens": 64,
126
+ "temperature": 1.1, #1.1 default
127
+ "return_full_text": False,
128
+ "do_sample": True,
129
+ },
130
+ "options":
131
+ {"use_cache": True,
132
+ "wait_for_model": True,
133
+ },}
134
+ response = requests.post(API_URL, headers=headers, json=json_)
135
+ #print(f"Response is : {response}")
136
+ output = response.json()
137
+ output_tmp = output[0]['generated_text']
138
+ print(f"Bloom API Response is : {output_tmp}")
139
+ if language == 'en':
140
+ solution = output_tmp.split("Answer: ")[2].split("\n")[0]
141
+ else:
142
+ output_tmp.split(".")[1]
143
+ print(f"Final Bloom Response after splits is: {solution}")
144
+ return solution
145
+
146
+ # Coqui - Text-to-Speech
147
+ def tts(text, language):
148
+ print(f"Inside tts - language is : {language}")
149
+ coqui_langs = ['en' ,'es' ,'fr' ,'de' ,'pl' ,'uk' ,'ro' ,'hu' ,'bg' ,'nl' ,'fi' ,'sl' ,'lv' ,'ga']
150
+ if language not in coqui_langs:
151
+ language = 'en'
152
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as fp:
153
+ coquiTTS.get_tts(text, fp, speaker = {"language" : language})
154
+ return fp.name
155
+
156
+ #demo = gr.Blocks()
157
+ #with demo:
158
+ # gr.Markdown("<h1><center>Testing</center></h1>")
159
+
160
+
161
+ gr.Interface(
162
+ title = 'Testing Whisper',
163
+ fn=driver_fun,
164
+ inputs=[
165
+ gr.Audio(source="microphone", type="filepath"), #streaming = True,
166
+ # "state"
167
+ ],
168
+ outputs=[
169
+ "textbox", "textbox", "textbox", "textbox", "audio",
170
+ ],
171
+ live=True).launch()