Spaces:
Runtime error
Runtime error
| import torch | |
| # os.system("pip install git+https://github.com/openai/whisper.git") | |
| import gradio as gr | |
| import whisper | |
| import librosa | |
| import plotly.express as px | |
| from threading import Thread | |
| from statistics import mode, mean | |
| import time | |
| model = whisper.load_model("medium", device='cpu') | |
| print('loaded whisper') | |
| vad, vad_utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', | |
| model='silero_vad', | |
| force_reload=False, | |
| onnx=False) | |
| print('loaded silero') | |
| (get_speech_timestamps, | |
| save_audio, | |
| read_audio, | |
| VADIterator, | |
| collect_chunks) = vad_utils | |
| vad_iterator = VADIterator(vad) | |
| global x, y, j, audio_vec, transcribe, STOP, languages, not_detected, main_lang, STARTED | |
| x = [] | |
| y = [] | |
| j = 0 | |
| STOP = False | |
| audio_vec = torch.tensor([]) | |
| transcribe = '' | |
| languages = [] | |
| not_detected = True | |
| main_lang = '' | |
| STARTED = False | |
| css = """ | |
| .gradio-container { | |
| font-family: 'IBM Plex Sans', sans-serif; | |
| } | |
| .gr-button { | |
| color: white; | |
| border-color: black; | |
| background: black; | |
| } | |
| input[type='range'] { | |
| accent-color: black; | |
| } | |
| .dark input[type='range'] { | |
| accent-color: #dfdfdf; | |
| } | |
| .container { | |
| max-width: 730px; | |
| margin: auto; | |
| padding-top: 1.5rem; | |
| } | |
| .details:hover { | |
| text-decoration: underline; | |
| } | |
| .gr-button { | |
| white-space: nowrap; | |
| } | |
| .gr-button:focus { | |
| border-color: rgb(147 197 253 / var(--tw-border-opacity)); | |
| outline: none; | |
| box-shadow: var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow, 0 0 #0000); | |
| --tw-border-opacity: 1; | |
| --tw-ring-offset-shadow: var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color); | |
| --tw-ring-shadow: var(--tw-ring-inset) 0 0 0 calc(3px var(--tw-ring-offset-width)) var(--tw-ring-color); | |
| --tw-ring-color: rgb(191 219 254 / var(--tw-ring-opacity)); | |
| --tw-ring-opacity: .5; | |
| } | |
| .footer { | |
| margin-bottom: 45px; | |
| margin-top: 35px; | |
| text-align: center; | |
| border-bottom: 1px solid #e5e5e5; | |
| } | |
| .footer>p { | |
| font-size: .8rem; | |
| display: inline-block; | |
| padding: 0 10px; | |
| transform: translateY(10px); | |
| background: white; | |
| } | |
| .dark .footer { | |
| border-color: #303030; | |
| } | |
| .dark .footer>p { | |
| background: #0b0f19; | |
| } | |
| .prompt h4{ | |
| margin: 1.25em 0 .25em 0; | |
| font-weight: bold; | |
| font-size: 115%; | |
| } | |
| .animate-spin { | |
| animation: spin 1s linear infinite; | |
| } | |
| @keyframes spin { | |
| from { | |
| transform: rotate(0deg); | |
| } | |
| to { | |
| transform: rotate(360deg); | |
| } | |
| } | |
| #share-btn-container { | |
| display: flex; margin-top: 1.5rem !important; padding-left: 0.5rem !important; padding-right: 0.5rem | |
| !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px | |
| !important; width: 13rem; | |
| } | |
| #share-btn { | |
| all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; | |
| margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important; | |
| } | |
| #share-btn * { | |
| all: unset; | |
| } | |
| """ | |
| # def transcribe_chunk(): | |
| # print('********************************') | |
| # global audio_vec, transcribe, STOP | |
| # print('Enter trans chunk') | |
| # counter = 0 | |
| # i = 0 | |
| # while not STOP: | |
| # if audio_vec.size()[0] // 32000 > counter and audio_vec.size()[0] > 0: | |
| # print('audio_vec.size()[0] % 32000', audio_vec.size()[0] % 32000) | |
| # print('audio size', audio_vec.size()[0]) | |
| # chunk = whisper.pad_or_trim(audio_vec[32000*counter: 32000*(counter + 1)]) | |
| # mel_th = whisper.log_mel_spectrogram(chunk).to(model.device) | |
| # options = whisper.DecodingOptions(fp16=False) | |
| # result = whisper.decode(model, mel_th, options) | |
| # no_speech_prob = result.no_speech_prob | |
| # if no_speech_prob < 0.4: | |
| # transcribe += result.text + ' ' | |
| # counter += 1 | |
| def transcribe_chunk(audio, vad_prob): | |
| global languages | |
| trnscrb = '' | |
| audio = whisper.pad_or_trim(audio) | |
| mel = whisper.log_mel_spectrogram(audio).to(model.device) | |
| options = whisper.DecodingOptions(fp16= False, task='transcribe') | |
| result = whisper.decode(model, mel, options) | |
| no_speech_prob = result.no_speech_prob | |
| mel = whisper.log_mel_spectrogram(audio).to(model.device) | |
| _, probs = model.detect_language(mel) | |
| temp_lang = max(probs, key=probs.get) | |
| print(result.text, "no_speech_prob: ",no_speech_prob, 1 - vad_prob) | |
| if no_speech_prob < 0.6: | |
| trnscrb = result.text + ' ' | |
| languages.append(temp_lang) | |
| if len(languages) > 3: | |
| languages.pop(0) | |
| return trnscrb | |
| def inference(audio): | |
| global x, y, j, audio_vec, transcribe, languages, not_detected, main_lang, STARTED | |
| print('enter inference') | |
| if j == 0: | |
| thread.start() | |
| STARTED = True | |
| wav2 = whisper.load_audio(audio, sr=16000) | |
| wav = torch.from_numpy(librosa.load(audio, sr=16000)[0]) | |
| audio_vec = torch.cat((audio_vec, wav)) | |
| speech_probs = [] | |
| window_size_samples = 1600 | |
| for i in range(0, len(wav), window_size_samples): | |
| chunk = wav[i: i + window_size_samples] | |
| if len(chunk) < window_size_samples: | |
| break | |
| speech_prob = vad(chunk, 16000).item() | |
| speech_probs.append(speech_prob) | |
| vad_iterator.reset_states() | |
| sample_per_sec = 16000 / window_size_samples | |
| x.extend([j + i / sample_per_sec for i in range(len(speech_probs))]) | |
| y.extend(speech_probs) | |
| j = max(x) | |
| fig = px.line(x=x, y=y) | |
| whisper_audio = whisper.pad_or_trim(wav2) | |
| mel = whisper.log_mel_spectrogram(whisper_audio).to(model.device) | |
| _, probs = model.detect_language(mel) | |
| temp_lang = max(probs, key=probs.get) | |
| print(temp_lang) | |
| languages.append(temp_lang) | |
| if len(languages) > 5: | |
| languages.pop(0) | |
| curr_lang = mode(languages) | |
| print(curr_lang, languages) | |
| if curr_lang == 'iw': | |
| return 'he', fig, gr.update(visible=True), transcribe, gr.update(visible=True), gr.update(visible=True) | |
| return curr_lang, fig, gr.update(visible=True), transcribe, gr.update(visible=True), gr.update(visible=True) | |
| def clear(): | |
| global x, y, j, audio_vec, transcribe, thread, STOP, languages, main_lang, not_detected ,STARTED | |
| STOP = True | |
| if STARTED: | |
| thread.join() | |
| STARTED = False | |
| x = [] | |
| y = [] | |
| j = 0 | |
| audio_vec = torch.tensor([]) | |
| transcribe = '' | |
| STOP = False | |
| languages = [] | |
| main_lang = '' | |
| not_detected = True | |
| thread = Thread(target=transcribe_chunk) | |
| print('clean:', x, y, j, transcribe, audio_vec) | |
| return '', gr.update(visible=False), gr.update(visible=False), '', gr.update(visible=False), gr.update(visible=False), | |
| def inference_file(audio): | |
| time.sleep(0.8) | |
| global x, y, j, audio_vec, transcribe, languages, not_detected, main_lang | |
| wav = torch.from_numpy(librosa.load(audio, sr=16000)[0]) | |
| audio_vec = torch.cat((audio_vec, wav)) | |
| speech_probs = [] | |
| window_size_samples = 1600 | |
| for i in range(0, len(wav), window_size_samples): | |
| chunk = wav[i: i + window_size_samples] | |
| if len(chunk) < window_size_samples: | |
| break | |
| speech_prob = vad(chunk, 16000).item() | |
| speech_probs.append(speech_prob) | |
| vad_iterator.reset_states() | |
| sample_per_sec = 16000 / window_size_samples | |
| x.extend([j + i / sample_per_sec for i in range(len(speech_probs))]) | |
| y.extend(speech_probs) | |
| j = max(x) | |
| fig = px.line(x=x, y=y) | |
| mean_speech_probs = mean(speech_probs) | |
| if wav.shape[0] > 16000 * 30: | |
| start = 0 | |
| end = 16000 * 30 | |
| chunk = wav[start:end] | |
| chunk_idx = 0 | |
| while end < wav.shape[0]: | |
| transcribe += transcribe_chunk(chunk) | |
| chunk_idx += 1 | |
| start = chunk_idx * 30 * 16000 | |
| if start >= wav.shape[0]: | |
| break | |
| end = (chunk_idx + 1) * 30 * 16000 | |
| if end >= wav.shape[0]: | |
| end = wav.shape[0] - 1 | |
| chunk = wav[start:end] | |
| else: | |
| transcribe += transcribe_chunk(wav, mean_speech_probs) | |
| curr_lang = '' | |
| if len(languages) > 0: | |
| curr_lang = mode(languages) | |
| print(curr_lang, languages) | |
| if curr_lang == 'iw': | |
| return 'he', fig, gr.update(visible=True), transcribe, gr.update(visible=True), gr.update(visible=True) | |
| return curr_lang, fig, gr.update(visible=True), transcribe, gr.update(visible=True), gr.update(visible=True) | |
| block = gr.Blocks(css=css) | |
| def play_sound(): | |
| global audio_vec | |
| import soundfile as sf | |
| print(audio_vec) | |
| sf.write('uploaded.wav', data=audio_vec, samplerate=16000) | |
| from pygame import mixer | |
| mixer.init() | |
| mixer.music.load('uploaded.wav') | |
| mixer.music.play() | |
| def change_audio(string): | |
| # if string == 'סטרימינג': | |
| # return gr.Audio.update(source="microphone",), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) | |
| # else: | |
| # return gr.Audio.update(source='upload'), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False) | |
| if string == 'סטרימינג': | |
| return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), \ | |
| gr.update(visible=False), gr.update(visible=False) | |
| elif string == 'הקלטה': | |
| print('in mesholav') | |
| return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), \ | |
| gr.update(visible=True), gr.update(visible=True) | |
| else: | |
| return gr.update(visible=False), gr.update(visible=True), gr.update(visible=True), \ | |
| gr.update(visible=False), gr.update(visible=False) | |
| with block: | |
| gr.HTML( | |
| """ | |
| <div style="text-align: center; max-width: 650px; margin: 0 auto;"> | |
| <div | |
| style=" | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 0.8rem; | |
| font-size: 1.75rem; | |
| " | |
| > | |
| <h1 style="font-weight: 900; margin-bottom: 7px;"> | |
| Whisper | |
| </h1> | |
| </div> | |
| </div> | |
| """ | |
| ) | |
| with gr.Group(): | |
| plot = gr.Plot(show_label=False, visible=False) | |
| with gr.Row(equal_height=True): | |
| with gr.Box(): | |
| radio = gr.Radio(["סטרימינג", "הקלטה", "קובץ"], label="?איך תרצה לספק את האודיו") | |
| with gr.Row().style(mobile_collapse=False, equal_height=True): | |
| audio = gr.Audio( | |
| show_label=False, | |
| source="microphone", | |
| type="filepath", | |
| visible=True | |
| ) | |
| audio2 = gr.Audio( | |
| label="Input Audio", | |
| show_label=False, | |
| source="upload", | |
| type="filepath", | |
| visible=False | |
| ) | |
| audio3 = gr.Audio( | |
| label="Input Audio", | |
| show_label=False, | |
| source="microphone", | |
| type="filepath", | |
| visible=False | |
| ) | |
| trans_btn = gr.Button("Transcribe", visible=False) | |
| trans_btn3 = gr.Button("Transcribe", visible=False) | |
| text = gr.Textbox(show_label=False, elem_id="result-textarea") | |
| text2 = gr.Textbox(show_label=False, elem_id="result-textarea") | |
| with gr.Row(): | |
| clear_btn = gr.Button("Clear", visible=False) | |
| play_btn = gr.Button('Play audio', visible=False) | |
| radio.change(fn=change_audio, inputs=radio, outputs=[audio, trans_btn, audio2, trans_btn3, audio3]) | |
| trans_btn.click(inference_file, audio2, [text, plot, plot, text2, clear_btn, play_btn]) | |
| trans_btn3.click(inference_file, audio3, [text, plot, plot, text2, clear_btn, play_btn]) | |
| audio.stream(inference_file, audio, [text, plot, plot, text2, clear_btn, play_btn]) | |
| play_btn.click(play_sound) | |
| clear_btn.click(clear, inputs=[], outputs=[text, plot, plot, text2, clear_btn, play_btn]) | |
| gr.HTML(''' | |
| <div class="footer"> | |
| <p>App by the best team - Ziv & Omer | |
| </p> | |
| </div> | |
| ''') | |
| gr.HTML(''' | |
| <img style="text-align: center; max-width: 650px; margin: 0 auto;" src="https://geekflare.com/wp-content/uploads/2022/02/speechrecognitionapi.png", alt="Girl in a jacket" width="500" height="600"> | |
| ''') | |
| global thread | |
| thread = Thread(target=transcribe_chunk) | |
| block.queue().launch() | |