innovation64 commited on
Commit
9c9550f
1 Parent(s): ae30e07

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +68 -0
  2. packages.txt +1 -0
  3. requirements.txt +4 -0
app.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import numpy as np
3
+ import torch
4
+ from datasets import load_dataset
5
+
6
+ from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline, VitsModel, VitsTokenizer
7
+
8
+
9
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
10
+
11
+ # load speech translation checkpoint
12
+ asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
13
+
14
+ # load text-to-speech checkpoint and speaker embeddings
15
+ model = VitsModel.from_pretrained("Matthijs/mms-tts-deu")
16
+ tokenizer = VitsTokenizer.from_pretrained("Matthijs/mms-tts-deu")
17
+
18
+
19
+ def translate(audio):
20
+ outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "de"})
21
+ return outputs["text"]
22
+
23
+
24
+ def synthesise(text):
25
+ inputs = tokenizer(text=text, return_tensors="pt")
26
+ speech_output = model(inputs["input_ids"].to(device))
27
+ speech = speech_output.audio[0]
28
+ return speech.cpu()
29
+
30
+
31
+ def speech_to_speech_translation(audio):
32
+ translated_text = translate(audio)
33
+ synthesised_speech = synthesise(translated_text)
34
+ synthesised_speech = (synthesised_speech.detach().numpy() * 32767).astype(np.int16)
35
+ return 16000, synthesised_speech
36
+
37
+
38
+ title = "Cascaded STST"
39
+ description = """
40
+ Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
41
+ [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
42
+
43
+ ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
44
+ """
45
+
46
+ demo = gr.Blocks()
47
+
48
+ mic_translate = gr.Interface(
49
+ fn=speech_to_speech_translation,
50
+ inputs=gr.Audio(source="microphone", type="filepath"),
51
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
52
+ title=title,
53
+ description=description,
54
+ )
55
+
56
+ file_translate = gr.Interface(
57
+ fn=speech_to_speech_translation,
58
+ inputs=gr.Audio(source="upload", type="filepath"),
59
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
60
+ examples=[["./example.wav"]],
61
+ title=title,
62
+ description=description,
63
+ )
64
+
65
+ with demo:
66
+ gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
67
+
68
+ demo.launch()
packages.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ ffmpeg
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ torch
2
+ git+https://github.com/hollance/transformers.git@6900e8ba6532162a8613d2270ec2286c3f58f57b
3
+ datasets
4
+ sentencepiece