import string import torch from transformers import pipeline from transformers.pipelines.audio_utils import ffmpeg_read import gradio as gr MODEL_NAME = "vinai/PhoWhisper-large" BATCH_SIZE = 8 device = 0 if torch.cuda.is_available() else "cpu" pipe = pipeline( task="automatic-speech-recognition", model=MODEL_NAME, chunk_length_s=30, device=device, ) desc = f""" This space transcribes Vietnamese words, phrases, and sentences via microphone or audio files then compares the user's text input to what the language model hears. You will then be given a PASS/FAIL grade to tell you if your spoken audio matches the text you entered. [{MODEL_NAME}](https://huggingface.co/{MODEL_NAME}) is the Vietnamese Speech-to-Text model that powers the analysis of the audio files. """ # Copied from https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/utils.py#L50 def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = "."): if seconds is not None: milliseconds = round(seconds * 1000.0) hours = milliseconds // 3_600_000 milliseconds -= hours * 3_600_000 minutes = milliseconds // 60_000 milliseconds -= minutes * 60_000 seconds = milliseconds // 1_000 milliseconds -= seconds * 1_000 hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else "" return f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}" else: # we have a malformed timestamp so just return it as is return seconds def transcribe(file, og_text, return_timestamps): outputs = pipe(file, batch_size=BATCH_SIZE, return_timestamps=return_timestamps) text = outputs["text"] if return_timestamps: timestamps = outputs["chunks"] timestamps = [ f"[{format_timestamp(chunk['timestamp'][0])} -> {format_timestamp(chunk['timestamp'][1])}] {chunk['text']}" for chunk in timestamps ] text = "\n".join(str(feature) for feature in timestamps) text_nopunc = text.translate(str.maketrans('', '', string.punctuation)) grade = '' if text_nopunc.lower() == og_text.lower(): grade = "good!" else: grade = "could use some work..." return text, grade demo = gr.Blocks() mic_transcribe = gr.Interface( fn=transcribe, inputs=[ gr.inputs.Audio(source="microphone", type="filepath", optional=True), gr.inputs.Textbox(label="Word/Phrase"), gr.inputs.Checkbox(default=False, label="Return timestamps"), ], outputs=[gr.Textbox(label="What I heard..."), gr.Textbox(label="Grade")], layout="vertical", theme="huggingface", title="Vietnamese Pronounciation Checker", description=(desc), allow_flagging="never", ) file_transcribe = gr.Interface( fn=transcribe, inputs=[ gr.inputs.Audio(source="upload", optional=True, label="Audio file", type="filepath"), gr.inputs.Textbox(label="Word/Phrase"), gr.inputs.Checkbox(default=False, label="Return timestamps"), ], outputs=[gr.Textbox(label="What I heard..."), gr.Textbox(label="Grade")], layout="vertical", theme="huggingface", title="Vietnamese Pronounciation Checker", description=(desc), examples=[ ["./example.flac", "transcribe", False], ["./example.flac", "transcribe", True], ], cache_examples=True, allow_flagging="never", ) with demo: gr.TabbedInterface([mic_transcribe, file_transcribe], ["Pronounce via Microphone", "Pronounce via Audio File"]) demo.launch(enable_queue=True)