import os import gradio as gr from pipelines.pipeline import InferencePipeline TITLE = """

Auto-AVSR: Audio-Visual Speech Recognition

Want to recognize content in a noisy environment?
Our Auto-AVSR models are here to transcribe your answers from audio or visual information!

""" ARTICLE = """

Want to look into models? You can find our [training code] and [paper].

The inference is performed on the CPU. You can also run on Colab GPU

We share this demo only for non-commercial purposes.

""" CSS = """ #col-container {margin-left: auto; margin-right: auto;} a {text-decoration-line: underline; font-weight: 600;} .animate-spin { animation: spin 1s linear infinite; } @keyframes spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } } #share-btn-container { display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem; } #share-btn { all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important; } #share-btn * { all: unset; } #share-btn-container div:nth-child(-n+2){ width: auto !important; min-height: 0px !important; } #share-btn-container .wrap { display: none !important; } """ pipelines = { "VSR(mediapipe)": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cpu", face_track=True, detector="mediapipe"), "ASR": InferencePipeline("./configs/LRS3_A_WER1.0.ini", device="cpu", face_track=True, detector="mediapipe"), "AVSR(mediapipe)": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cpu", face_track=True, detector="mediapipe") } def fn(pipeline_type, filename): selected_pipeline_instance = pipelines[pipeline_type] landmarks = selected_pipeline_instance.process_landmarks(filename, landmarks_filename=None) data = selected_pipeline_instance.dataloader.load_data(filename, landmarks) transcript = selected_pipeline_instance.model.infer(data) return transcript demo = gr.Blocks(css=CSS) with demo: gr.HTML(TITLE) dropdown_list = gr.inputs.Dropdown(["ASR", "VSR(mediapipe)", "AVSR(mediapipe)"], label="model") video_file = gr.Video(label="INPUT VIDEO", include_audio=True) text = gr.Textbox(label="PREDICTION") btn = gr.Button("Submit").style(full_width=True) btn.click(fn, inputs=[dropdown_list, video_file], outputs=text) gr.HTML(ARTICLE) demo.launch()