Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
### PRE ###
|
2 |
+
import os
|
3 |
+
os.system('git clone https://github.com/ggerganov/whisper.cpp.git')
|
4 |
+
os.system('make -C ./whisper.cpp')
|
5 |
+
MODELS_TO_DOWNLOAD = ['tiny', 'medium'] # ['tiny', 'small', 'base', 'medium', 'large']
|
6 |
+
|
7 |
+
for model_name in MODELS_TO_DOWNLOAD:
|
8 |
+
os.system(f'bash ./whisper.cpp/models/download-ggml-model.sh {model_name}')
|
9 |
+
|
10 |
+
|
11 |
+
### BODY ###
|
12 |
+
|
13 |
+
import os
|
14 |
+
import requests
|
15 |
+
import json
|
16 |
+
import base64
|
17 |
+
|
18 |
+
import gradio as gr
|
19 |
+
from pathlib import Path
|
20 |
+
import pysrt
|
21 |
+
import pandas as pd
|
22 |
+
import re
|
23 |
+
import time
|
24 |
+
|
25 |
+
from pytube import YouTube
|
26 |
+
import torch
|
27 |
+
|
28 |
+
whisper_models = MODELS_TO_DOWNLOAD #["medium"]#["base", "small", "medium", "large", "base.en"]
|
29 |
+
|
30 |
+
custom_models = []
|
31 |
+
combined_models = []
|
32 |
+
combined_models.extend(whisper_models)
|
33 |
+
combined_models.extend(custom_models)
|
34 |
+
|
35 |
+
LANGUAGES = {
|
36 |
+
"bg": "Bulgarian",
|
37 |
+
}
|
38 |
+
|
39 |
+
# language code lookup by name, with a few language aliases
|
40 |
+
source_languages = {
|
41 |
+
**{language: code for code, language in LANGUAGES.items()}
|
42 |
+
}
|
43 |
+
|
44 |
+
source_language_list = [key[0] for key in source_languages.items()]
|
45 |
+
|
46 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
47 |
+
print(f"DEVICE IS: {device}")
|
48 |
+
|
49 |
+
def get_youtube(video_url):
|
50 |
+
yt = YouTube(video_url)
|
51 |
+
abs_video_path = yt.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first().download()
|
52 |
+
print(f"Download complete - {abs_video_path}")
|
53 |
+
return abs_video_path
|
54 |
+
|
55 |
+
def speech_to_text(video_file_path, selected_source_lang, whisper_model):
|
56 |
+
"""
|
57 |
+
Speech Recognition is based on models from OpenAI Whisper https://github.com/openai/whisper
|
58 |
+
This space is using c++ implementation by https://github.com/ggerganov/whisper.cpp
|
59 |
+
"""
|
60 |
+
|
61 |
+
if(video_file_path == None):
|
62 |
+
raise ValueError("Error no video input")
|
63 |
+
|
64 |
+
print(video_file_path)
|
65 |
+
_,file_ending = os.path.splitext(f'{video_file_path}')
|
66 |
+
input_wav_file = video_file_path.replace(file_ending, ".wav")
|
67 |
+
srt_path = input_wav_file + ".srt"
|
68 |
+
vtt_path = input_wav_file + ".vtt"
|
69 |
+
try:
|
70 |
+
print(f'file enging is {file_ending}, starting conversion to wav')
|
71 |
+
subs_paths = video_file_path.replace(file_ending, ".wav")
|
72 |
+
|
73 |
+
if os.path.exists(subs_paths):
|
74 |
+
os.remove(subs_paths)
|
75 |
+
|
76 |
+
os.system(f'ffmpeg -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{subs_paths}"')
|
77 |
+
print("conversion to wav ready")
|
78 |
+
|
79 |
+
except Exception as e:
|
80 |
+
raise RuntimeError("Error Running inference with local model", e)
|
81 |
+
|
82 |
+
try:
|
83 |
+
print("starting whisper c++")
|
84 |
+
os.system(f'rm -f {srt_path}')
|
85 |
+
print('Running regular model')
|
86 |
+
os.system(f'./whisper.cpp/main "{input_wav_file}" -t {os.cpu_count()} -l {source_languages.get(selected_source_lang)} -m ./whisper.cpp/models/ggml-{whisper_model}.bin -osrt -ovtt')
|
87 |
+
print("whisper c++ finished")
|
88 |
+
except Exception as e:
|
89 |
+
raise RuntimeError("Error running Whisper cpp model")
|
90 |
+
|
91 |
+
print(f'Subtitles path {srt_path}, {vtt_path}')
|
92 |
+
return [vtt_path, srt_path]
|
93 |
+
|
94 |
+
def create_video_player(subs_files, video_in):
|
95 |
+
print(f"create_video_player - {subs_files}, {video_in}")
|
96 |
+
|
97 |
+
with open(subs_files[0], "rb") as file:
|
98 |
+
subtitle_base64 = base64.b64encode(file.read())
|
99 |
+
|
100 |
+
with open(video_in, "rb") as file:
|
101 |
+
video_base64 = base64.b64encode(file.read())
|
102 |
+
|
103 |
+
video_player = f'''<video id="video" controls preload="metadata">
|
104 |
+
<source src="data:video/mp4;base64,{str(video_base64)[2:-1]}" type="video/mp4" />
|
105 |
+
<track
|
106 |
+
label="English"
|
107 |
+
kind="subtitles"
|
108 |
+
srclang="en"
|
109 |
+
src="data:text/vtt;base64,{str(subtitle_base64)[2:-1]}"
|
110 |
+
default />
|
111 |
+
</video>
|
112 |
+
'''
|
113 |
+
|
114 |
+
print('create_video_player - Done')
|
115 |
+
return video_player
|
116 |
+
|
117 |
+
|
118 |
+
# ---- Gradio Layout -----
|
119 |
+
video_in = gr.Video(label="Video file", mirror_webcam=False)
|
120 |
+
youtube_url_in = gr.Textbox(label="Youtube url", lines=1, interactive=True)
|
121 |
+
video_out = gr.Video(label="Video Out", mirror_webcam=False)
|
122 |
+
|
123 |
+
selected_source_lang = gr.Dropdown(choices=source_language_list,
|
124 |
+
type="value",
|
125 |
+
value= source_language_list[0], #"Let the model analyze",
|
126 |
+
label="Spoken language in video",
|
127 |
+
interactive=True)
|
128 |
+
selected_whisper_model = gr.Dropdown(choices=whisper_models,
|
129 |
+
type="value",
|
130 |
+
value=whisper_models[0],#"base",
|
131 |
+
label="Selected Whisper model",
|
132 |
+
interactive=True)
|
133 |
+
|
134 |
+
subtitle_files = gr.File(
|
135 |
+
label="Download subtitles",
|
136 |
+
file_count="multiple",
|
137 |
+
type="file",
|
138 |
+
interactive=False,
|
139 |
+
)
|
140 |
+
|
141 |
+
video_player = gr.HTML('<p>video will be played here after you press the button at step 4')
|
142 |
+
eventslider = gr.Slider(visible=False)
|
143 |
+
status_msg = gr.Markdown('Status')
|
144 |
+
|
145 |
+
demo = gr.Blocks()
|
146 |
+
demo.encrypt = False
|
147 |
+
|
148 |
+
def set_app_msg(app_state, msg):
|
149 |
+
app_state['status_msg'] = msg
|
150 |
+
|
151 |
+
def transcribe(app_state, youtube_url_in, selected_source_lang, selected_whisper_model):
|
152 |
+
set_app_msg(app_state, 'Downloading the movie ...')
|
153 |
+
video_file_path = get_youtube(youtube_url_in)
|
154 |
+
set_app_msg(app_state, f'Running the speech to text model {selected_source_lang}/{selected_whisper_model}. This can take some time.')
|
155 |
+
subtitle_files = speech_to_text(video_file_path, selected_source_lang, selected_whisper_model)
|
156 |
+
set_app_msg(app_state, f'Creating the video player ...')
|
157 |
+
video_player = create_video_player(subtitle_files, video_file_path)
|
158 |
+
set_app_msg(app_state, f'Done...')
|
159 |
+
return subtitle_files, video_player
|
160 |
+
|
161 |
+
|
162 |
+
def on_change_event(app_state):
|
163 |
+
print('Running!')
|
164 |
+
return app_state['status_msg']
|
165 |
+
|
166 |
+
with demo:
|
167 |
+
app_state = gr.State({
|
168 |
+
'running':False,
|
169 |
+
'status_msg': ''
|
170 |
+
})
|
171 |
+
|
172 |
+
with gr.Row():
|
173 |
+
with gr.Column():
|
174 |
+
gr.Markdown('''### 1. Copy any non-private Youtube video URL to box below or click one of the examples.''')
|
175 |
+
examples = gr.Examples(examples=["https://www.youtube.com/watch?v=UjAn3Pza3qo", "https://www.youtube.com/watch?v=oOZivhYfPD4"],
|
176 |
+
label="Examples", inputs=[youtube_url_in])
|
177 |
+
# Inspiration from https://huggingface.co/spaces/vumichien/whisper-speaker-diarization
|
178 |
+
|
179 |
+
with gr.Row():
|
180 |
+
with gr.Column():
|
181 |
+
youtube_url_in.render()
|
182 |
+
selected_source_lang.render()
|
183 |
+
selected_whisper_model.render()
|
184 |
+
|
185 |
+
download_youtube_btn = gr.Button("Transcribe the video")
|
186 |
+
download_youtube_btn.click(transcribe, [app_state, youtube_url_in, selected_source_lang, selected_whisper_model], [subtitle_files, video_player])
|
187 |
+
|
188 |
+
eventslider.render()
|
189 |
+
status_msg.render()
|
190 |
+
subtitle_files.render()
|
191 |
+
video_player.render()
|
192 |
+
with gr.Row():
|
193 |
+
gr.Markdown('This app is based on [this code](https://huggingface.co/spaces/RASMUS/Whisper-youtube-crosslingual-subtitles/tree/main) by RASMUS.')
|
194 |
+
|
195 |
+
dep = demo.load(on_change_event, inputs=[app_state], outputs=[status_msg], every=10)
|
196 |
+
|
197 |
+
|
198 |
+
|
199 |
+
#### RUN ###
|
200 |
+
is_kaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE')
|
201 |
+
print(is_kaggle)
|
202 |
+
|
203 |
+
if is_kaggle:
|
204 |
+
demo.queue().launch(share=True, debug=True)
|
205 |
+
else:
|
206 |
+
demo.queue().launch()
|
207 |
+
|