Marco-Cheung sanchit-gandhi HF staff commited on
Commit
0a5f7db
0 Parent(s):

Duplicate from course-demos/speech-to-speech-translation

Browse files

Co-authored-by: Sanchit Gandhi <sanchit-gandhi@users.noreply.huggingface.co>

Files changed (6) hide show
  1. .gitattributes +35 -0
  2. README.md +13 -0
  3. app.py +72 -0
  4. example.wav +0 -0
  5. packages.txt +1 -0
  6. requirements.txt +4 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Speech To Speech Translation
3
+ emoji: 🏆
4
+ colorFrom: pink
5
+ colorTo: indigo
6
+ sdk: gradio
7
+ sdk_version: 3.36.1
8
+ app_file: app.py
9
+ pinned: false
10
+ duplicated_from: course-demos/speech-to-speech-translation
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import numpy as np
3
+ import torch
4
+ from datasets import load_dataset
5
+
6
+ from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline
7
+
8
+
9
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
10
+
11
+ # load speech translation checkpoint
12
+ asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-base", device=device)
13
+
14
+ # load text-to-speech checkpoint and speaker embeddings
15
+ processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
16
+
17
+ model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts").to(device)
18
+ vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(device)
19
+
20
+ embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
21
+ speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
22
+
23
+
24
+ def translate(audio):
25
+ outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "translate"})
26
+ return outputs["text"]
27
+
28
+
29
+ def synthesise(text):
30
+ inputs = processor(text=text, return_tensors="pt")
31
+ speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
32
+ return speech.cpu()
33
+
34
+
35
+ def speech_to_speech_translation(audio):
36
+ translated_text = translate(audio)
37
+ synthesised_speech = synthesise(translated_text)
38
+ synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
39
+ return 16000, synthesised_speech
40
+
41
+
42
+ title = "Cascaded STST"
43
+ description = """
44
+ Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
45
+ [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
46
+
47
+ ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
48
+ """
49
+
50
+ demo = gr.Blocks()
51
+
52
+ mic_translate = gr.Interface(
53
+ fn=speech_to_speech_translation,
54
+ inputs=gr.Audio(source="microphone", type="filepath"),
55
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
56
+ title=title,
57
+ description=description,
58
+ )
59
+
60
+ file_translate = gr.Interface(
61
+ fn=speech_to_speech_translation,
62
+ inputs=gr.Audio(source="upload", type="filepath"),
63
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
64
+ examples=[["./example.wav"]],
65
+ title=title,
66
+ description=description,
67
+ )
68
+
69
+ with demo:
70
+ gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
71
+
72
+ demo.launch()
example.wav ADDED
Binary file (263 kB). View file
 
packages.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ ffmpeg
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ torch
2
+ git+https://github.com/huggingface/transformers
3
+ datasets
4
+ sentencepiece