Spaces:
Runtime error
Runtime error
import os | |
import gradio as gr | |
from pipelines.pipeline import InferencePipeline | |
pipelines = { | |
"VSR(mediapipe)": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cpu", face_track=True, detector="mediapipe"), | |
"ASR": InferencePipeline("./configs/LRS3_A_WER1.0.ini", device="cpu", face_track=True, detector="mediapipe"), | |
"AVSR(mediapipe)": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cpu", face_track=True, detector="mediapipe") | |
} | |
print("Step 0. Model has been loaded.") | |
def fn(pipeline_type, filename): | |
print("Step 0. Video has been uploaded.") | |
selected_pipeline_instance = pipelines[pipeline_type] | |
print("Step 1. Video has been converted.") | |
landmarks = selected_pipeline_instance.process_landmarks(filename, landmarks_filename=None) | |
print("Step 2. Landmarks have been detected.") | |
data = selected_pipeline_instance.dataloader.load_data(filename, landmarks) | |
print("Step 3. Data has been preprocessed.") | |
transcript = selected_pipeline_instance.model.infer(data) | |
print("Step 4. Inference has been done.") | |
print(f"transcript: {transcript}") | |
return transcript | |
demo = gr.Blocks() | |
with demo: | |
gr.HTML( | |
""" | |
<div style="text-align: center; max-width: 1200px; margin: 20px auto;"> | |
<h1 style="font-weight: 900; font-size: 3rem; margin: 0rem"> | |
Auto-AVSR | |
</h1> | |
<h3 style="font-weight: 450; font-size: 1rem; margin: 0rem"> | |
[<a href="https://arxiv.org/abs/2303.14307" style="color:blue;">arXiv</a>] | |
[<a href="https://github.com/mpc001/auto_avsr" style="color:blue;">Code</a>] | |
</h3> | |
<h2 style="text-align: left; font-weight: 450; font-size: 1rem; margin-top: 0.5rem; margin-bottom: 0.5rem"> | |
🔥 <b>Notes</b>: We share this demo only for non-commercial purposes. | |
</h2> | |
</div> | |
""") | |
dropdown_list = gr.inputs.Dropdown(["ASR", "VSR(mediapipe)", "AVSR(mediapipe)"], label="model") | |
video_file = gr.Video(label="INPUT VIDEO", include_audio=True) | |
text = gr.Textbox(label="PREDICTION") | |
btn = gr.Button("Submit").style(full_width=True) | |
btn.click(fn, inputs=[dropdown_list, video_file], outputs=text) | |
demo.launch() |