update_fs2 (#3)

- add spn_predictor (cc4865c44418f8dcc33f73a53b41e122cb092565)
- add newest model (e7867f340bb1a94cf23ebad46fcbeb20a64e12b6)


Co-authored-by: Yingzhi Wang <yingzhi@users.noreply.huggingface.co>

README.md

@@ -16,13 +16,11 @@ metrics:
 <iframe src="https://ghbtns.com/github-btn.html?user=speechbrain&repo=speechbrain&type=star&count=true&size=large&v=2" frameborder="0" scrolling="0" width="170" height="30" title="GitHub"></iframe>
 <br/><br/>
 
-**IMPORTANT: This is a work in progress. This model is not providing meaningful output at the moment**
-
 # Text-to-Speech (TTS) with FastSpeech2 trained on LJSpeech
 
 This repository provides all the necessary tools for Text-to-Speech (TTS)  with SpeechBrain using a [FastSpeech2](https://arxiv.org/abs/2006.04558) pretrained on [LJSpeech](https://keithito.com/LJ-Speech-Dataset/).
 
-The pre-trained model takes in input a short text and produces a spectrogram in output. One can get the final waveform by applying a vocoder (e.g., HiFIGAN) on top of the generated spectrogram.
+The pre-trained model takes texts or phonemes as input and produces a spectrogram in output. One can get the final waveform by applying a vocoder (e.g., HiFIGAN) on top of the generated spectrogram. It should be noted that if the input is text, we use a state-of-the-art grapheme-to-phoneme module to convert it to phonemes and then pass the phonemes to fastspeech2 model.
 
 
 ## Install SpeechBrain
@@ -46,16 +44,28 @@ from speechbrain.pretrained import HIFIGAN
 
 # Intialize TTS (tacotron2) and Vocoder (HiFIGAN)
 fastspeech2 = FastSpeech2.from_hparams(source="speechbrain/tts-fastspeech2-ljspeech", savedir="tmpdir_tts")
-hifi_gan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-libritts-16kHz", savedir="tmpdir_vocoder")
+hifi_gan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-ljspeech", savedir="tmpdir_vocoder")
+
+# Run TTS with text input
+input_text = "were the leaders in this luckless change; though our own Baskerville; who was at work some years before them; went much on the same lines;"
+mel_output, durations, pitch, energy = fastspeech2.encode_text([input_text])
+
+# Running Vocoder (spectrogram-to-waveform)
+waveforms = hifi_gan.decode_batch(mel_output)
+
+# Save the waverform
+torchaudio.save('example_TTS_input_text.wav', waveforms.squeeze(1), 22050)
+
 
-# Running the TTS
-mel_output, durations, pitch, energy = fastspeech2.encode_text(input_text)
+# Run TTS with phoneme input
+input_phonemes = ['W', 'ER', 'DH', 'AH', 'L', 'IY', 'D', 'ER', 'Z', 'IH', 'N', 'DH', 'IH', 'S', 'L', 'AH', 'K', 'L', 'AH', 'S', 'CH', 'EY', 'N', 'JH', 'spn', 'DH', 'OW', 'AW', 'ER', 'OW', 'N', 'B', 'AE', 'S', 'K', 'ER', 'V', 'IH', 'L', 'spn', 'HH', 'UW', 'W', 'AA', 'Z', 'AE', 'T', 'W', 'ER', 'K', 'S', 'AH', 'M', 'Y', 'IH', 'R', 'Z', 'B', 'IH', 'F', 'AO', 'R', 'DH', 'EH', 'M', 'spn', 'W', 'EH', 'N', 'T', 'M', 'AH', 'CH', 'AA', 'N', 'DH', 'AH', 'S', 'EY', 'M', 'L', 'AY', 'N', 'Z', 'spn']
+mel_output, durations, pitch, energy = fastspeech2.encode_phoneme([input_phonemes])
 
 # Running Vocoder (spectrogram-to-waveform)
 waveforms = hifi_gan.decode_batch(mel_output)
 
 # Save the waverform
-torchaudio.save('example_TTS.wav', waveforms.squeeze(1), 16000)
+torchaudio.save('example_TTS_input_phoneme.wav', waveforms.squeeze(1), 22050)
 ```
 
 If you want to generate multiple sentences in one-shot, you can do in this way:
@@ -68,7 +78,7 @@ items = [
        "How much wood would a woodchuck chuck?",
        "Never odd or even"
      ]
-mel_outputs, durations, pitch, energy = fastspeech2.encode_batch(items)
+mel_outputs, durations, pitch, energy = fastspeech2.encode_text(items)
 
 ```
 

config.json

@@ -1,5 +1,5 @@
 {
   "speechbrain_interface": "FastSpeech2",
   "vocoder_interface": "HiFIGAN",
-  "vocoder_model_id": "speechbrain/tts-hifigan-libritts-16kHz"
+  "vocoder_model_id": "speechbrain/tts-hifigan-ljspeech"
 }

hyperparams.yaml

@@ -45,26 +45,26 @@ lexicon:
     - ZH
     - spn
 
-n_symbols: 41 #fixed deppending on symbols in the lexicon +1 for a dummy symbol used for padding
+n_symbols: 42 #fixed deppending on symbols in the lexicon +1 for a dummy symbol used for padding
 padding_idx: 0
 n_mel_channels: 80
 
 # Encoder parameters
 enc_num_layers: 4
 enc_num_head: 2
-enc_d_model: 256
+enc_d_model: 384
 enc_ffn_dim: 1024
-enc_k_dim: 256
-enc_v_dim: 256
+enc_k_dim: 384
+enc_v_dim: 384
 enc_dropout: 0.2
 
 # Decoder parameters
 dec_num_layers: 4
 dec_num_head: 2
-dec_d_model: 256
+dec_d_model: 384
 dec_ffn_dim: 1024
-dec_k_dim: 256
-dec_v_dim: 256
+dec_k_dim: 384
+dec_v_dim: 384
 dec_dropout: 0.2
 
 # Postnet parameters
@@ -84,6 +84,21 @@ pitch_pred_kernel_size: 3
 energy_pred_kernel_size: 3
 variance_predictor_dropout: 0.5
 
+# silent phoneme token predictor
+spn_predictor: !new:speechbrain.lobes.models.FastSpeech2.SPNPredictor
+    enc_num_layers: !ref <enc_num_layers>
+    enc_num_head: !ref <enc_num_head>
+    enc_d_model: !ref <enc_d_model>
+    enc_ffn_dim: !ref <enc_ffn_dim>
+    enc_k_dim: !ref <enc_k_dim>
+    enc_v_dim: !ref <enc_v_dim>
+    enc_dropout: !ref <enc_dropout>
+    normalize_before: !ref <normalize_before>
+    ffn_type: !ref <ffn_type>
+    ffn_cnn_kernel_size_list: !ref <ffn_cnn_kernel_size_list>
+    n_char: !ref <n_symbols>
+    padding_idx: !ref <padding_idx>
+
 # Model
 model: !new:speechbrain.lobes.models.FastSpeech2.FastSpeech2
     enc_num_layers: !ref <enc_num_layers>
@@ -119,8 +134,10 @@ model: !new:speechbrain.lobes.models.FastSpeech2.FastSpeech2
 input_encoder: !new:speechbrain.dataio.encoder.TextEncoder
 
 modules:
+    spn_predictor: !ref <spn_predictor>
     model: !ref <model>
 
 pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
     loadables:
+        spn_predictor: !ref <spn_predictor>
         model: !ref <model>

model.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dfc40f7ad123936bb18f56d6f2392198c09b50eb416fdfb0895ec2077f4ed6cc
-size 114702155
+oid sha256:ed731151171fa256527ef2d1ced5dda7b4cae46d630ca2f4a0f9f0b1ff627429
+size 173137489

spn_predictor.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a7b0cc08a811489a086d49bfa96772eabc302d972925e2d6693831641a21f00
+size 72515155