Spaces:
Runtime error
Runtime error
File size: 3,315 Bytes
a35325d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2023 Imperial College London (Pingchuan Ma)
# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
import os
import datetime
import subprocess
import gradio as gr
from pipelines.pipeline import InferencePipeline
FFMPEG_COMMAND = "-loglevel error -y -r 25 -pix_fmt yuv420p -f mp4"
pipelines = {
"VSR": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cuda:0", face_track=True, detector="retinaface"),
"VSR(fast)": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cuda:0", face_track=True, detector="mediapipe"),
"ASR": InferencePipeline("./configs/LRS3_A_WER1.0.ini", device="cuda:0", face_track=True, detector="retinaface"),
"AVSR": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cuda:0", face_track=True, detector="retinaface"),
"AVSR(fast)": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cuda:0", face_track=True, detector="mediapipe")
}
print("Step 0. Model has been loaded.")
def fn(pipeline_type, filename):
directory = "./tmp_video"
if not os.path.exists(directory):
os.makedirs(directory)
now = datetime.datetime.now()
timestamp = now.strftime("%Y-%m-%d_%H-%M-%S")
dst_filename = f"{directory}/file_{timestamp}.mp4"
command_string = f"ffmpeg -i {filename} {FFMPEG_COMMAND} {dst_filename}"
print("Step 0. Video has been uploaded.")
os.system(command_string)
selected_pipeline_instance = pipelines[pipeline_type]
print("Step 1. Video has been converted.")
landmarks = selected_pipeline_instance.process_landmarks(dst_filename, landmarks_filename=None)
print("Step 2. Landmarks have been detected.")
data = selected_pipeline_instance.dataloader.load_data(dst_filename, landmarks)
print("Step 3. Data has been preprocessed.")
transcript = selected_pipeline_instance.model.infer(data)
print("Step 4. Inference has been done.")
print(f"transcript: {transcript}")
return transcript
demo = gr.Blocks()
with demo:
gr.HTML(
"""
<div style="text-align: center; max-width: 700px; margin: 0 auto;">
<div
style="
display: inline-flex;
align-items: center;
gap: 0.8rem;
font-size: 1.75rem;
"
>
<h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels
</h1>
</div>
<p style="margin-bottom: 10px; font-size: 94%">
</p>
</div>
"""
)
dropdown_list = gr.inputs.Dropdown(["VSR", "ASR", "AVSR", "VSR(fast)", "AVSR(fast)"], label="model")
video_file = gr.Video(label="INPUT VIDEO", include_audio=True)
text = gr.Textbox(label="PREDICTION")
btn = gr.Button("Submit").style(full_width=True)
btn.click(fn, inputs=[dropdown_list, video_file], outputs=text)
with gr.Accordion("Additional information", open=False):
gr.HTML(
"""
<div class="acknowledgments">
<p> We share this demo only for non-commercial purposes. </p>
</div>
"""
)
demo.launch(share=True) |