pragsGit commited on
Commit
ea99259
1 Parent(s): 4f5ae06

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +72 -0
app.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import numpy as np
3
+ import torch
4
+ from datasets import load_dataset
5
+
6
+ from transformers import pipeline
7
+ from transformers import VitsModel, VitsTokenizer
8
+
9
+
10
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
11
+
12
+ # load speech translation checkpoint
13
+ asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
14
+
15
+
16
+
17
+ model = VitsModel.from_pretrained("facebook/mms-tts-spa")
18
+ processor = VitsTokenizer.from_pretrained("facebook/mms-tts-spa")
19
+
20
+
21
+
22
+
23
+ def translate(audio):
24
+ outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"language": "es","task": "transcribe"})
25
+ return outputs["text"]
26
+
27
+
28
+ def synthesise(text):
29
+ inputs = processor(text=text, return_tensors="pt")
30
+ with torch.no_grad():
31
+ speech = model(inputs["input_ids"].to(device))
32
+ return speech.audio[0]
33
+
34
+
35
+ def speech_to_speech_translation(audio):
36
+ translated_text = translate(audio)
37
+ synthesised_speech = synthesise(translated_text)
38
+ synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
39
+ return 16000, synthesised_speech
40
+
41
+
42
+ title = "Cascaded STST"
43
+ description = """
44
+ Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
45
+ [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
46
+
47
+ ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
48
+ """
49
+
50
+ demo = gr.Blocks()
51
+
52
+ mic_translate = gr.Interface(
53
+ fn=speech_to_speech_translation,
54
+ inputs=gr.Audio(source="microphone", type="filepath"),
55
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
56
+ title=title,
57
+ description=description,
58
+ )
59
+
60
+ file_translate = gr.Interface(
61
+ fn=speech_to_speech_translation,
62
+ inputs=gr.Audio(source="upload", type="filepath"),
63
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
64
+ examples=[["./example.wav"]],
65
+ title=title,
66
+ description=description,
67
+ )
68
+
69
+ with demo:
70
+ gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
71
+
72
+ demo.launch()