Sandiago21 commited on
Commit
8a0926c
1 Parent(s): 6c02a88

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. README.md +3 -9
  2. app.py +99 -0
  3. example.wav +0 -0
  4. packages.txt +2 -0
  5. requirements.txt +6 -0
README.md CHANGED
@@ -1,12 +1,6 @@
1
  ---
2
- title: Speech To Speech Translation Spanish
3
- emoji: 🏢
4
- colorFrom: red
5
- colorTo: pink
6
- sdk: gradio
7
- sdk_version: 3.36.1
8
  app_file: app.py
9
- pinned: false
 
10
  ---
11
-
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: speech-to-speech-translation-spanish
 
 
 
 
 
3
  app_file: app.py
4
+ sdk: gradio
5
+ sdk_version: 3.36.0
6
  ---
 
 
app.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import numpy as np
3
+ import torch
4
+ from datasets import load_dataset
5
+ from transformers import SpeechT5ForTextToSpeech, SpeechT5HifiGan, SpeechT5Processor, pipeline
6
+
7
+
8
+ device = "cuda:0" if torch.cuda.is_available() else "cpu"
9
+
10
+ # load speech translation checkpoint
11
+ asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v2", device=device)
12
+
13
+ # load text-to-speech checkpoint and speaker embeddings
14
+ model_id = "Sandiago21/speecht5_finetuned_voxpopuli_spanish" # update with your model id
15
+ # pipe = pipeline("automatic-speech-recognition", model=model_id)
16
+ model = SpeechT5ForTextToSpeech.from_pretrained(model_id)
17
+ vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
18
+ embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
19
+ speaker_embeddings = torch.tensor(embeddings_dataset[7440]["xvector"]).unsqueeze(0)
20
+
21
+ processor = SpeechT5Processor.from_pretrained(model_id)
22
+
23
+ replacements = [
24
+ ("á", "a"),
25
+ ("ç", "c"),
26
+ ("è", "e"),
27
+ ("ì", "i"),
28
+ ("í", "i"),
29
+ ("ò", "o"),
30
+ ("ó", "o"),
31
+ ("ù", "u"),
32
+ ("ú", "u"),
33
+ ("š", "s"),
34
+ ("ï", "i")
35
+ ("ñ", "n"),
36
+ ("ü", "u"),
37
+ ]
38
+
39
+ def cleanup_text(text):
40
+ for src, dst in replacements:
41
+ text = text.replace(src, dst)
42
+ return text
43
+
44
+ def synthesize_speech(text):
45
+ text = cleanup_text(text)
46
+ inputs = processor(text=text, return_tensors="pt")
47
+ speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
48
+
49
+ return gr.Audio.update(value=(16000, speech.cpu().numpy()))
50
+
51
+ def translate(audio):
52
+ outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "italian"})
53
+ return outputs["text"]
54
+
55
+
56
+ def synthesise(text):
57
+ text = cleanup_text(text)
58
+ inputs = processor(text=text, return_tensors="pt")
59
+ speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
60
+ return speech.cpu()
61
+
62
+
63
+ def speech_to_speech_translation(audio):
64
+ translated_text = translate(audio)
65
+ synthesised_speech = synthesise(translated_text)
66
+ synthesised_speech = (synthesised_speech.numpy() * 32767).astype(np.int16)
67
+ return 16000, synthesised_speech
68
+
69
+
70
+ title = "Cascaded STST"
71
+ description = """
72
+ Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Spanish. Demo uses OpenAI's [Whisper Large v2](https://huggingface.co/openai/whisper-large-v2) model for speech translation, and [Sandiago21/speecht5_finetuned_voxpopuli_spanish](https://huggingface.co/Sandiago21/speecht5_finetuned_voxpopuli_spanish) checkpoint for text-to-speech, which is based on Microsoft's
73
+ [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech, fine-tuned in Spanish Audio dataset:
74
+ ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
75
+ """
76
+
77
+ demo = gr.Blocks()
78
+
79
+ mic_translate = gr.Interface(
80
+ fn=speech_to_speech_translation,
81
+ inputs=gr.Audio(source="microphone", type="filepath"),
82
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
83
+ title=title,
84
+ description=description,
85
+ )
86
+
87
+ file_translate = gr.Interface(
88
+ fn=speech_to_speech_translation,
89
+ inputs=gr.Audio(source="upload", type="filepath"),
90
+ outputs=gr.Audio(label="Generated Speech", type="numpy"),
91
+ examples=[["./example.wav"]],
92
+ title=title,
93
+ description=description,
94
+ )
95
+
96
+ with demo:
97
+ gr.TabbedInterface([mic_translate, file_translate], ["Microphone", "Audio File"])
98
+
99
+ demo.launch()
example.wav ADDED
Binary file (603 kB). View file
 
packages.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ ffmpeg
2
+
requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ torch
2
+ git+https://github.com/huggingface/transformers
3
+ datasets
4
+ torchaudio
5
+ sentencepiece
6
+