Spaces:
Runtime error
Runtime error
File size: 2,235 Bytes
a35325d d577e1e a35325d f1fc904 a35325d f1fc904 a35325d 8fa7a2d a35325d 8fa7a2d 4c034e2 a35325d 557008d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
import os
import gradio as gr
from pipelines.pipeline import InferencePipeline
pipelines = {
"VSR(mediapipe)": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cpu", face_track=True, detector="mediapipe"),
"ASR": InferencePipeline("./configs/LRS3_A_WER1.0.ini", device="cpu", face_track=True, detector="mediapipe"),
"AVSR(mediapipe)": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cpu", face_track=True, detector="mediapipe")
}
print("Step 0. Model has been loaded.")
def fn(pipeline_type, filename):
print("Step 0. Video has been uploaded.")
selected_pipeline_instance = pipelines[pipeline_type]
print("Step 1. Video has been converted.")
landmarks = selected_pipeline_instance.process_landmarks(filename, landmarks_filename=None)
print("Step 2. Landmarks have been detected.")
data = selected_pipeline_instance.dataloader.load_data(filename, landmarks)
print("Step 3. Data has been preprocessed.")
transcript = selected_pipeline_instance.model.infer(data)
print("Step 4. Inference has been done.")
print(f"transcript: {transcript}")
return transcript
demo = gr.Blocks()
with demo:
gr.HTML(
"""
<div style="text-align: center; max-width: 1200px; margin: 20px auto;">
<h1 style="font-weight: 900; font-size: 3rem; margin: 0rem">
Auto-AVSR
</h1>
<h3 style="font-weight: 450; font-size: 1rem; margin: 0rem">
[<a href="https://arxiv.org/abs/2303.14307" style="color:blue;">arXiv</a>]
[<a href="https://github.com/mpc001/auto_avsr" style="color:blue;">Code</a>]
</h3>
<h2 style="text-align: left; font-weight: 450; font-size: 1rem; margin-top: 0.5rem; margin-bottom: 0.5rem">
🔥 <b>Notes</b>: We share this demo only for non-commercial purposes.
</h2>
</div>
""")
dropdown_list = gr.inputs.Dropdown(["ASR", "VSR(mediapipe)", "AVSR(mediapipe)"], label="model")
video_file = gr.Video(label="INPUT VIDEO", include_audio=True)
text = gr.Textbox(label="PREDICTION")
btn = gr.Button("Submit").style(full_width=True)
btn.click(fn, inputs=[dropdown_list, video_file], outputs=text)
demo.launch() |