Update tacotron model that uses phonemes instead of raw text.

.gitattributes

@@ -27,9 +27,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 bazelisk-linux-amd64 filter=lfs diff=lfs merge=lfs -text
 wavegru_mod.so filter=lfs diff=lfs merge=lfs -text
-pretrained_model_ljs_600k.ckpt filter=lfs diff=lfs merge=lfs -text
-wavegru_vocoder_1024_v3_1310000.ckpt filter=lfs diff=lfs merge=lfs -text
-wavegru_vocoder_1024_v3_1330000.ckpt filter=lfs diff=lfs merge=lfs -text
-wavegru_vocoder_1024_v3_1340000.ckpt filter=lfs diff=lfs merge=lfs -text
-wavegru_vocoder_1024_v3_1360000.ckpt filter=lfs diff=lfs merge=lfs -text
-wavegru_vocoder_1024_v3_1400000.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.venv

alphabet.txt

@@ -1,25 +1,18 @@
 _
+■
  
 !
 "
-'
-(
-)
 ,
--
 .
 :
 ;
 ?
-[
-]
 a
 b
-c
 d
 e
 f
-g
 h
 i
 j
@@ -29,7 +22,6 @@ m
 n
 o
 p
-q
 r
 s
 t
@@ -37,5 +29,29 @@ u
 v
 w
 x
-y
 z
+æ
+ð
+ŋ
+ɐ
+ɑ
+ɔ
+ə
+ɚ
+ɛ
+ɜ
+ɡ
+ɪ
+ɹ
+ɾ
+ʃ
+ʊ
+ʌ
+ʒ
+ʔ
+ˈ
+ˌ
+ː
+̩
+θ
+ᵻ

app.py

@@ -3,6 +3,10 @@
 # os.system("./bazelisk-linux-amd64 clean --expunge")
 # os.system("./bazelisk-linux-amd64 build wavegru_mod -c opt --copt=-march=native")
 
+# install espeak
+import os
+
+os.system("bash ./install_espeak_ng.sh")
 
 import gradio as gr
 from inference import load_tacotron_model, load_wavegru_net, mel_to_wav, text_to_mel
@@ -11,7 +15,7 @@ from wavegru_cpp import extract_weight_mask, load_wavegru_cpp
 
 def speak(text):
     alphabet, tacotron_net, tacotron_config = load_tacotron_model(
-        "./alphabet.txt", "./tacotron.toml", "./pretrained_model_ljs_600k.ckpt"
+        "./alphabet.txt", "./tacotron.toml", "./tacotrons_ljs_24k_v1_0250000.ckpt"
     )
 
     wavegru_config, wavegru_net = load_wavegru_net(

inference.py

@@ -1,3 +1,5 @@
+import os
+
 import jax
 import jax.numpy as jnp
 import librosa
@@ -14,6 +16,11 @@ from utils import (
 )
 from wavegru import WaveGRU
 
+os.environ["PHONEMIZER_ESPEAK_LIBRARY"] = "./espeak/usr/lib/libespeak-ng.so.1.1.51"
+from phonemizer.backend import EspeakBackend
+
+backend = EspeakBackend("en-us", preserve_punctuation=True, with_stress=True)
+
 
 def load_tacotron_model(alphabet_file, config_file, model_file):
     """load tacotron model to memory"""
@@ -34,6 +41,8 @@ tacotron_inference_fn = pax.pure(lambda net, text: net.inference(text, max_len=2
 def text_to_mel(net, text, alphabet, config):
     """convert text to mel spectrogram"""
     text = english_cleaners(text)
+    text = backend.phonemize([text], strip=True)[0]
+    text = text + config["END_CHARACTER"]
     text = text + config["PAD"] * (100 - (len(text) % 100))
     tokens = []
     for c in text:

install_espeak_ng.sh

@@ -0,0 +1,10 @@
+rm -rf espeak
+mkdir -p espeak
+cd espeak
+wget https://github.com/espeak-ng/espeak-ng/archive/refs/tags/1.51.zip
+unzip -qq 1.51.zip
+cd espeak-ng-1.51
+./autogen.sh
+./configure --prefix=`pwd`/../usr
+make
+make install

packages.txt

@@ -1 +1,7 @@
 libsndfile1-dev
+make
+autoconf
+automake
+libtool
+pkg-config
+gcc

requirements.txt

@@ -8,4 +8,5 @@ numpy==1.22.3
 pax3==0.5.6
 pyyaml==6.0
 toml==0.10.2
-unidecode==1.3.4
+unidecode==1.3.4
+phonemizer==3.1.1

tacotron.py

@@ -371,7 +371,10 @@ class Tacotron(pax.Module):
         x = x[:, : self.rr, :]
         x = jnp.reshape(x, (N, self.rr, -1))
         mel = x[..., :-1]
-        eos = x[..., -1]
+        eos_logit = x[..., -1]
+        eos_pr = jax.nn.sigmoid(eos_logit[0, -1])
+        rng_key, eos_rng_key = jax.random.split(rng_key)
+        eos = jax.random.bernoulli(eos_rng_key, p=eos_pr)
         return attn_state, decoder_rnn_states, rng_key, (mel, eos)
 
     def inference(self, text, seed=42, max_len=1000):
@@ -381,6 +384,7 @@ class Tacotron(pax.Module):
         text = self.encode_text(text)
         text_key = self.text_key_fc(text)
         N, L, D = text.shape
+        assert N == 1
         mel = self.go_frame(N)
 
         attn_state, decoder_rnn_states = self.decoder_initial_state(N, L)
@@ -393,7 +397,7 @@ class Tacotron(pax.Module):
                 attn_state, decoder_rnn_states, rng_key, mel, text, text_key
             )
             mels.append(mel)
-            if eos[0, -1].item() > 0 or count > max_len:
+            if eos.item() or count > max_len:
                 break
 
             mel = mel[:, -1, :]

tacotron.toml

@@ -16,6 +16,7 @@ MEL_DIM = 80 # the dimension of melspectrogram features
 MEL_MIN = 1e-5
 PAD = "_" # padding character
 PAD_TOKEN = 0
+END_CHARACTER = "■"  # to signal the end of the transcript
 TEST_DATA_SIZE = 1024
 
 # model

pretrained_model_ljs_600k.ckpt → tacotrons_ljs_24k_v1_0250000.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ec09df89ab9b0e1fd0e310e2888d8dd3590a6dd60d2c6a6ff5c378016f5f381
-size 53525995
+oid sha256:512b3af6ef95ccc53d3516256abae81b025e110fa886ec68f9f7033039013fc6
+size 53561547