Update app.py

app.py

@@ -1,101 +1,71 @@
+import gradio as gr
+import numpy as np
 import torch
+from datasets import load_dataset
+
 from transformers import pipeline
-from transformers.pipelines.audio_utils import ffmpeg_read
-import gradio as gr
+from transformers import VitsModel, VitsTokenizer
 
-MODEL_NAME = "openai/whisper-small"
-BATCH_SIZE = 8
 
-device = 0 if torch.cuda.is_available() else "cpu"
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+# load speech translation checkpoint
+asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
+
+
+
+model = VitsModel.from_pretrained("facebook/mms-tts-spa")
+processor = VitsTokenizer.from_pretrained("facebook/mms-tts-spa")
 
-pipe = pipeline(
-    task="automatic-speech-recognition",
-    model=MODEL_NAME,
-    chunk_length_s=30,
-    device=device,
-)
 
 
-# Copied from https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/utils.py#L50
-def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = "."):
-    if seconds is not None:
-        milliseconds = round(seconds * 1000.0)
 
-        hours = milliseconds // 3_600_000
-        milliseconds -= hours * 3_600_000
+def translate(audio):
+    outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"language": "es","task": "transcribe"})
+    return outputs["text"]
 
-        minutes = milliseconds // 60_000
-        milliseconds -= minutes * 60_000
 
-        seconds = milliseconds // 1_000
-        milliseconds -= seconds * 1_000
+def synthesise(text):
+    inputs = processor(text=text, return_tensors="pt")
+    with torch.no_grad():
+      speech = model(inputs["input_ids"].to(device))
+    return speech.audio[0]
 
-        hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else ""
-        return f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}"
-    else:
-        # we have a malformed timestamp so just return it as is
-        return seconds
 
+def speech_to_speech_translation(audio):
+    translated_text = translate(audio)
+    synthesised_speech = synthesise(translated_text)
+    synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
+    return 16000, synthesised_speech
 
-def transcribe(file, task, return_timestamps):
-    outputs = pipe(file, batch_size=BATCH_SIZE, generate_kwargs={"task": task}, return_timestamps=return_timestamps)
-    text = outputs["text"]
-    if return_timestamps:
-        timestamps = outputs["chunks"]
-        timestamps = [
-            f"[{format_timestamp(chunk['timestamp'][0])} -> {format_timestamp(chunk['timestamp'][1])}] {chunk['text']}"
-            for chunk in timestamps
-        ]
-        text = "\n".join(str(feature) for feature in timestamps)
-    return text
 
+title = "Cascaded STST"
+description = """
+Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
+[SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
+![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
+"""
 
 demo = gr.Blocks()
 
-mic_transcribe = gr.Interface(
-    fn=transcribe,
-    inputs=[
-        gr.inputs.Audio(source="microphone", type="filepath", optional=True),
-        gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
-        gr.inputs.Checkbox(default=False, label="Return timestamps"),
-    ],
-    outputs="text",
-    layout="horizontal",
-    theme="huggingface",
-    title="Whisper Demo: Transcribe Audio",
-    description=(
-        "Transcribe long-form microphone or audio inputs with the click of a button! Demo uses the"
-        f" checkpoint [{MODEL_NAME}](https://huggingface.co/{MODEL_NAME}) and 🤗 Transformers to transcribe audio files"
-        " of arbitrary length."
-    ),
-    allow_flagging="never",
+mic_translate = gr.Interface(
+    fn=speech_to_speech_translation,
+    inputs=gr.Audio(source="microphone", type="filepath"),
+    outputs=gr.Audio(label="Generated Speech", type="numpy"),
+    title=title,
+    description=description,
 )
 
-file_transcribe = gr.Interface(
-    fn=transcribe,
-    inputs=[
-        gr.inputs.Audio(source="upload", optional=True, label="Audio file", type="filepath"),
-        gr.inputs.Radio(["transcribe", "translate"], label="Task", default="transcribe"),
-        gr.inputs.Checkbox(default=False, label="Return timestamps"),
-    ],
-    outputs="text",
-    layout="horizontal",
-    theme="huggingface",
-    title="Whisper Demo: Transcribe Audio",
-    description=(
-        "Transcribe long-form microphone or audio inputs with the click of a button! Demo uses the"
-        f" checkpoint [{MODEL_NAME}](https://huggingface.co/{MODEL_NAME}) and 🤗 Transformers to transcribe audio files"
-        " of arbitrary length."
-    ),
-    examples=[
-        ["./example.flac", "transcribe", False],
-        ["./example.flac", "transcribe", True],
-    ],
-    cache_examples=True,
-    allow_flagging="never",
+file_translate = gr.Interface(
+    fn=speech_to_speech_translation,
+    inputs=gr.Audio(source="upload", type="filepath"),
+    outputs=gr.Audio(label="Generated Speech", type="numpy"),
+    examples=[["./example.wav"]],
+    title=title,
+    description=description,
 )
 
 with demo:
-    gr.TabbedInterface([mic_transcribe, file_transcribe], ["Transcribe Microphone", "Transcribe Audio File"])
+    gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
 
-demo.launch(enable_queue=True)
+demo.launch()