File size: 2,220 Bytes
d347764 9307dab d347764 bfd1577 d347764 73eab05 d347764 bfd1577 d347764 c9dc5f3 1864517 d347764 1864517 bfd1577 d347764 f805e49 bfd1577 f805e49 c737803 d347764 bcd35fa d347764 f805e49 d347764 c737803 bcd35fa c737803 3946ba6 c737803 d347764 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
import gradio as gr
import numpy as np
import torch
from datasets import load_dataset
from transformers import VitsModel, VitsTokenizer
from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# load speech translation checkpoint
asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-en-ru")
model = VitsModel.from_pretrained('facebook/mms-tts-rus').to(device)
tokenizer = VitsTokenizer.from_pretrained('facebook/mms-tts-rus')
def translate(audio):
outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "translate"})
return translator(outputs['text'])[0]['translation_text']
def synthesise(text):
inputs = tokenizer(text=text, return_tensors="pt")
with torch.no_grad():
speech = model(**inputs).waveform
return speech.cpu()
def speech_to_speech_translation(audio):
translated_text = translate(audio)
synthesised_speech = synthesise(translated_text)
synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
return 16000, synthesised_speech[0]
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Russian
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
demo.launch()
|