Add vocos.

app.py

@@ -6,6 +6,7 @@ import gradio as gr
 import numpy as np
 
 import torch
+import json
 
 
 from hifigan.config import v1
@@ -19,17 +20,23 @@ from pflow.text import  text_to_sequence, sequence_to_text
 from pflow.utils.utils import intersperse
 from pflow.data.text_mel_datamodule import mel_spectrogram
 from pflow.utils.model import normalize
+from vocos import Vocos
 
 
 
-PFLOW_MODEL_PATH = 'checkpoint_epoch=649.ckpt'
-VOCODER_MODEL_PATH = 'g_00140000_m'
+
+
+PFLOW_MODEL_PATH = 'checkpoints/checkpoint_epoch=649.ckpt'
+#PFLOW_MODEL_PATH = 'checkpoint_m_epoch=054.ckpt'
+VOCODER_MODEL_PATH = 'checkpoints/pytorch_model.bin'
+HIFIGAN_MODEL_PATH = 'checkpoints/g_00120000'
+
 
 transform = torchaudio.transforms.Vol(gain=-32, gain_type="db")
 wav, sr = torchaudio.load('prompt.wav')
 
 prompt = mel_spectrogram(
-            transform(wav),
+            wav,
             1024,
             80,
             22050,
@@ -42,6 +49,7 @@ prompt = mel_spectrogram(
 
 
 
+
 def process_text(text: str, device: torch.device):
     x = torch.tensor(
         intersperse(text_to_sequence(text, ["ukr_cleaners"]), 0),
@@ -65,12 +73,25 @@ def load_hifigan(checkpoint_path, device):
 
 
 
+
+def load_vocos(checkpoint_path, config_path, device):
+    model = Vocos.from_hparams(config_path)
+
+    raw_model = torch.load(checkpoint_path, map_location=torch.device('cpu'))
+    raw_model = raw_model if 'state_dict' not in raw_model else raw_model['state_dict']
+    model.load_state_dict(raw_model, strict=False)
+    model.eval()
+    return model
+
+
 def to_waveform(mel, vocoder, denoiser=None):
-    audio = vocoder(mel).clamp(-1, 1)
-    if denoiser is not None:
-        audio = denoiser(audio.squeeze(), strength=0.00025).cpu().squeeze()
+    return  vocoder.decode(mel).cpu().squeeze()
 
-    return audio.cpu().squeeze()
+    # audio = vocoder(mel).clamp(-1, 1)
+    # if denoiser is not None:
+    #     audio = denoiser(audio.squeeze(), strength=0.00025).cpu().squeeze()
+
+    # return audio.cpu().squeeze()
 
 
 
@@ -90,11 +111,16 @@ def get_device():
 device = get_device()
 model = pflowTTS.load_from_checkpoint(PFLOW_MODEL_PATH, map_location=device)
 _ = model.eval()
-vocoder = load_hifigan(VOCODER_MODEL_PATH, device)
-denoiser = Denoiser(vocoder, mode="zeros")
+
+
+#hifigan = load_hifigan(HIFIGAN_MODEL_PATH, device)
+vocos = load_vocos(VOCODER_MODEL_PATH, 'config.yaml', device)
+#vocos_44100 = load_vocos('checkpoints/vocos_checkpoint_epoch=4_step=93440_val_loss=5.2596_44100_10.ckpt', 'vocos.yaml', device)
+denoiser = None#Denoiser(vocoder, mode="zeros")
+
 
 @torch.inference_mode()
-def synthesise(text, temperature, speed):
+def synthesise(text, speed):
     if len(text) > 1000:
         raise gr.Error("Текст повинен бути коротшим за 1000 символів.")
 
@@ -104,21 +130,27 @@ def synthesise(text, temperature, speed):
         text_processed["x"].to(device),
         text_processed["x_lengths"].to(device),
         n_timesteps=40,
-        temperature=temperature,
+        temperature=0.0,
         length_scale=1/speed,
         prompt=normalize(prompt, model.mel_mean, model.mel_std).to(device),
-        guidance_scale=1.0
+
+        guidance_scale=1.5
+
     )
-    waveform = to_waveform(output["mel"], vocoder, denoiser)
+    waveform_vocos = vocos.decode(output["mel"]).cpu().squeeze()
+    #waveform_vocos_44100 = vocos_44100.decode(output["mel"]).cpu().squeeze()
+    #waveform_hifigan = hifigan(output["mel"]).clamp(-1, 1).cpu().squeeze()
+    #transform = torchaudio.transforms.Vol(gain=-18, gain_type="db")
+
 
-    return text_processed['x_phones'][1::2], (22050, waveform.numpy())
+    return text_processed['x_phones'][1::2], (22050, waveform_vocos.numpy())
 
 
 description = f'''
 # Експериментальна апка для генерації аудіо з тексту.
 
     pflow checkpoint {PFLOW_MODEL_PATH}
-    vocoder: HIFIGAN(трейнутий на датасеті, з нуля) - {VOCODER_MODEL_PATH}
+    vocoder: Vocos - {VOCODER_MODEL_PATH}
 '''
 
 
@@ -128,17 +160,28 @@ if __name__ == "__main__":
         description=description,
         inputs=[
             gr.Text(label='Текст для синтезу:', lines=5, max_lines=10),
-            gr.Slider(minimum=0.0, maximum=1.0, label="Температура", value=0.4),
             gr.Slider(minimum=0.6, maximum=2.0, label="Швидкість", value=1.0)
         ],
         outputs=[
             gr.Text(label='Фонемізований текст:', lines=5),
+            # gr.Audio(
+            #             label="Vocos 44100 аудіо:",
+            #             autoplay=False,
+            #             streaming=False,
+            #             type="numpy",
+            #         ),
             gr.Audio(
-                        label="Згенероване аудіо:",
+                        label="Vocos аудіо:",
                         autoplay=False,
                         streaming=False,
                         type="numpy",
-                    )
+                    ),
+            # gr.Audio(
+            #             label="HIFIGAN аудіо:",
+            #             autoplay=False,
+            #             streaming=False,
+            #             type="numpy",
+            #         )
             
         ],
         allow_flagging ='manual',

checkpoint_epoch=499.ckpt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:39051170c6c0d9abce47d0073f796912d5ce3854ade8f707cb30333f50160d99
-size 279562867

checkpoint_epoch=649.ckpt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f86bc69330121d97d876f8cc38a8f7c36c443be40b2b0b4389b9684d4c351c6a
-size 279563122

g_00120000 → checkpoints/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f25c6dbc515ed387edd5d2e5683a50510aa33986e8a79273efe1216084f0f078
-size 55824433
+oid sha256:0af7b6f4b153819ada44a917135acf33944cdbb70cde0701eda3d100153799c7
+size 54051047

config.yaml

@@ -0,0 +1,33 @@
+# pytorch_lightning==1.8.6
+
+feature_extractor:
+  class_path: vocos.feature_extractors.MelSpectrogramFeatures
+  init_args:
+    sample_rate: 22050
+    n_fft: 1024
+    hop_length: 256
+    n_mels: 80
+    padding: same
+    f_min: 0
+    f_max: 8000
+    norm: "slaney"
+    mel_scale: "slaney"
+
+
+backbone:
+  class_path: vocos.models.VocosBackbone
+  init_args:
+    input_channels: 80
+    dim: 512
+    intermediate_dim: 1536
+    num_layers: 8
+
+head:
+  class_path: vocos.heads.ISTFTHead
+  init_args:
+    dim: 512
+    n_fft: 1024
+    hop_length: 256
+    padding: same
+
+

g_00140000_m

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c4edf30ac1bbb52cd250f0c38d615df23978d39ee8415c2a4c636344367adfd1
-size 55824433