ckandemir commited on
Commit
835f922
1 Parent(s): 4dc27cf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +18 -12
app.py CHANGED
@@ -1,4 +1,3 @@
1
-
2
  import gradio as gr
3
  import numpy as np
4
  import torch
@@ -12,7 +11,7 @@ device = "cuda:0" if torch.cuda.is_available() else "cpu"
12
  asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v2", device=device)
13
 
14
  # load text-to-speech checkpoint and speaker embeddings
15
- model_id = "ckandemir/speecht5_finetuned_voxpopuli_fr" # update with your model id
16
  # pipe = pipeline("automatic-speech-recognition", model=model_id)
17
  model = SpeechT5ForTextToSpeech.from_pretrained(model_id)
18
  vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
@@ -22,12 +21,19 @@ speaker_embeddings = torch.tensor(embeddings_dataset[7440]["xvector"]).unsqueeze
22
  processor = SpeechT5Processor.from_pretrained(model_id)
23
 
24
  replacements = [
25
- ("à", "a"), ("â", "a"),
26
- ("ç", "c"),
27
- ("é", "e"), ("è", "e"), ("ê", "e"), ("ë", "e"),
28
- ("î", "i"), ("ï", "i"),
29
- ("ô", "o"),
30
- ("ù", "u"), ("û", "u"),
 
 
 
 
 
 
 
31
  ]
32
 
33
  def cleanup_text(text):
@@ -43,7 +49,7 @@ def synthesize_speech(text):
43
  return gr.Audio.update(value=(16000, speech.cpu().numpy()))
44
 
45
  def translate(audio):
46
- outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "french"})
47
  return outputs["text"]
48
 
49
 
@@ -61,13 +67,13 @@ def speech_to_speech_translation(audio):
61
  return 16000, synthesised_speech
62
 
63
 
64
-
65
  title = "Cascaded STST"
66
  description = """
67
- Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in French. Demo uses OpenAI's [Whisper Large v2](https://huggingface.co/openai/whisper-large-v2) model for speech translation, and [ckandemir/speecht5_finetuned_voxpopuli_fr](https://huggingface.co/ckandemir/speecht5_finetuned_voxpopuli_fr) checkpoint for text-to-speech, which is based on Microsoft's
68
- [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech, fine-tuned in French Audio dataset:
69
  ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
70
  """
 
71
  demo = gr.Blocks()
72
 
73
  mic_translate = gr.Interface(
 
 
1
  import gradio as gr
2
  import numpy as np
3
  import torch
 
11
  asr_pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large-v2", device=device)
12
 
13
  # load text-to-speech checkpoint and speaker embeddings
14
+ model_id = "Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish" # update with your model id
15
  # pipe = pipeline("automatic-speech-recognition", model=model_id)
16
  model = SpeechT5ForTextToSpeech.from_pretrained(model_id)
17
  vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
 
21
  processor = SpeechT5Processor.from_pretrained(model_id)
22
 
23
  replacements = [
24
+ ("á", "a"),
25
+ ("ç", "c"),
26
+ ("è", "e"),
27
+ ("ì", "i"),
28
+ ("í", "i"),
29
+ ("ò", "o"),
30
+ ("ó", "o"),
31
+ ("ù", "u"),
32
+ ("ú", "u"),
33
+ ("š", "s"),
34
+ ("ï", "i"),
35
+ ("ñ", "n"),
36
+ ("ü", "u"),
37
  ]
38
 
39
  def cleanup_text(text):
 
49
  return gr.Audio.update(value=(16000, speech.cpu().numpy()))
50
 
51
  def translate(audio):
52
+ outputs = asr_pipe(audio, max_new_tokens=256, generate_kwargs={"task": "transcribe", "language": "spanish"})
53
  return outputs["text"]
54
 
55
 
 
67
  return 16000, synthesised_speech
68
 
69
 
 
70
  title = "Cascaded STST"
71
  description = """
72
+ Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in Spanish. Demo uses OpenAI's [Whisper Large v2](https://huggingface.co/openai/whisper-large-v2) model for speech translation, and [Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish](https://huggingface.co/Sandiago21/speecht5_finetuned_facebook_voxpopuli_spanish) checkpoint for text-to-speech, which is based on Microsoft's
73
+ [SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech, fine-tuned in Spanish Audio dataset:
74
  ![Cascaded STST](https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png "Diagram of cascaded speech to speech translation")
75
  """
76
+
77
  demo = gr.Blocks()
78
 
79
  mic_translate = gr.Interface(