Spaces:
Runtime error
Runtime error
File size: 2,483 Bytes
d347764 9fd1f21 d347764 9fd1f21 d347764 9fd1f21 d347764 9fd1f21 d347764 9fd1f21 d347764 9fd1f21 d347764 9fd1f21 d347764 f805e49 9fd1f21 f805e49 c737803 d347764 226ec3a d347764 f805e49 d347764 c737803 9fd1f21 c737803 3946ba6 c737803 d347764 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
import gradio as gr
import numpy as np
import torch
from transformers import AutoTokenizer, VitsModel
from transformers import pipeline
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# Translate audio to russian text
asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-tiny", device=device)
translator_to_ru = pipeline("translation", model="Helsinki-NLP/opus-mt-en-ru")
def translate(audio, translator_to_ru: pipeline = translator_to_ru):
outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "translate"})
return translator_to_ru(outputs['text'])[0]['translation_text']
# Text to russian speech
model = VitsModel.from_pretrained("facebook/mms-tts-rus")
tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-rus")
def synthesise(text: str, tokenizer: AutoTokenizer = tokenizer, model: VitsModel = model):
inputs = tokenizer(text, return_tensors="pt")
# print(inputs)
with torch.no_grad():
output = model(**inputs).waveform
return output.cpu()
def speech_to_speech_translation(audio):
translated_text = translate(audio)
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
return 16000, synthesised_speech[0]
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in multi language to target speech in Russian. Demo uses OpenAI's [Whisper Tiny](https://huggingface.co/openai/whisper-tiny) model for speech translation, and Facebook's
[mms-tts-rus](https://huggingface.co/acebook/mms-tts-rus) model for text-to-speech:
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./test_2.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
demo.launch()
|