mix

app.py

@@ -1,3 +1,5 @@
+import sys,os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from vits.models import SynthesizerInfer
 from omegaconf import OmegaConf
 import torchcrepe
@@ -72,7 +74,7 @@ model = SynthesizerInfer(
     hp.data.filter_length // 2 + 1,
     hp.data.segment_size // hp.data.hop_length,
     hp)
-load_svc_model("vits_pretrain/sovits5.0_bigvgan.pth", model)
+load_svc_model("vits_pretrain/sovits5.0_bigvgan_mix.pth", model)
 model.eval()
 model.to(device)
 
@@ -81,6 +83,8 @@ def svc_change(argswave, argsspk):
 
     argsppg = "svc_tmp.ppg.npy"
     os.system(f"python whisper/inference.py -w {argswave} -p {argsppg}")
+    argsvec = "svc_tmp.vec.npy"
+    os.system(f"python hubert/inference.py -w {argswave} -v {argsvec}")
 
     spk = np.load(argsspk)
     spk = torch.FloatTensor(spk)
@@ -89,13 +93,20 @@ def svc_change(argswave, argsspk):
     ppg = np.repeat(ppg, 2, 0)  # 320 PPG -> 160 * 2
     ppg = torch.FloatTensor(ppg)
 
+    vec = np.load(argsvec)
+    vec = np.repeat(vec, 2, 0)  # 320 PPG -> 160 * 2
+    vec = torch.FloatTensor(vec)
+
     pit = compute_f0_nn(argswave, device)
     pit = torch.FloatTensor(pit)
 
     len_pit = pit.size()[0]
+    len_vec = vec.size()[0]
     len_ppg = ppg.size()[0]
-    len_min = min(len_pit, len_ppg)
+    len_min = min(len_pit, len_vec)
+    len_min = min(len_min, len_ppg)
     pit = pit[:len_min]
+    vec = vec[:len_min, :]
     ppg = ppg[:len_min, :]
 
     with torch.no_grad():
@@ -129,11 +140,12 @@ def svc_change(argswave, argsspk):
                 cut_e_out = -1 * hop_frame * hop_size
 
             sub_ppg = ppg[cut_s:cut_e, :].unsqueeze(0).to(device)
+            sub_vec = vec[cut_s:cut_e, :].unsqueeze(0).to(device)
             sub_pit = pit[cut_s:cut_e].unsqueeze(0).to(device)
             sub_len = torch.LongTensor([cut_e - cut_s]).to(device)
             sub_har = source[:, :, cut_s *
                              hop_size:cut_e * hop_size].to(device)
-            sub_out = model.inference(sub_ppg, sub_pit, spk, sub_len, sub_har)
+            sub_out = model.inference(sub_ppg, sub_vec, sub_pit, spk, sub_len, sub_har)
             sub_out = sub_out[0, 0].data.cpu().detach().numpy()
 
             sub_out = sub_out[cut_s_out:cut_e_out]
@@ -148,10 +160,11 @@ def svc_change(argswave, argsspk):
                 cut_s = 0
                 cut_s_out = 0
             sub_ppg = ppg[cut_s:, :].unsqueeze(0).to(device)
+            sub_vec = vec[cut_s:, :].unsqueeze(0).to(device)
             sub_pit = pit[cut_s:].unsqueeze(0).to(device)
             sub_len = torch.LongTensor([all_frame - cut_s]).to(device)
             sub_har = source[:, :, cut_s * hop_size:].to(device)
-            sub_out = model.inference(sub_ppg, sub_pit, spk, sub_len, sub_har)
+            sub_out = model.inference(sub_ppg, sub_vec, sub_pit, spk, sub_len, sub_har)
             sub_out = sub_out[0, 0].data.cpu().detach().numpy()
 
             sub_out = sub_out[cut_s_out:]
@@ -187,6 +200,8 @@ with app:
                 
                 https://github.com/Multi-Singer/Multi-Singer.github.io
 
+                mix_encoder: whisper + hubert, 提升跨语言能力和纯对白语音训练的效果
+
                 [轻度伴奏可以无需去伴奏]就能直接进行歌声转换的SVC库
                 """)
             sid = gr.Dropdown(label="音色", choices=[

configs/base.yaml

@@ -28,11 +28,12 @@ data:
 #############################
 vits:
   ppg_dim: 1024
+  vec_dim: 256
   spk_dim: 256
   gin_channels: 256
   inter_channels: 192
   hidden_channels: 192
-  filter_channels: 512
+  filter_channels: 640
 #############################
 gen:
   upsample_input: 192

vits/data_utils.py

@@ -29,13 +29,15 @@ class TextAudioSpeakerSet(torch.utils.data.Dataset):
         items_new = []
         items_min = int(self.segment_size / self.hop_length * 4)  # 1 S
         items_max = int(self.segment_size / self.hop_length * 16)  # 4 S
-        for wavpath, spec, pitch, ppg, spk in self.items:
+        for wavpath, spec, pitch, vec, ppg, spk in self.items:
             if not os.path.isfile(wavpath):
                 continue
             if not os.path.isfile(spec):
                 continue
             if not os.path.isfile(pitch):
                 continue
+            if not os.path.isfile(vec):
+                continue
             if not os.path.isfile(ppg):
                 continue
             if not os.path.isfile(spk):
@@ -46,7 +48,7 @@ class TextAudioSpeakerSet(torch.utils.data.Dataset):
                 continue
             if (usel >= items_max):
                 usel = items_max
-            items_new.append([wavpath, spec, pitch, ppg, spk, usel])
+            items_new.append([wavpath, spec, pitch, vec, ppg, spk, usel])
             lengths.append(usel)
         self.items = items_new
         self.lengths = lengths
@@ -70,28 +72,35 @@ class TextAudioSpeakerSet(torch.utils.data.Dataset):
         wav = item[0]
         spe = item[1]
         pit = item[2]
-        ppg = item[3]
-        spk = item[4]
-        use = item[5]
+        vec = item[3]
+        ppg = item[4]
+        spk = item[5]
+        use = item[6]
 
         wav = self.read_wav(wav)
         spe = torch.load(spe)
 
         pit = np.load(pit)
+        vec = np.load(vec)
+        vec = np.repeat(vec, 2, 0)  # 320 PPG -> 160 * 2
         ppg = np.load(ppg)
         ppg = np.repeat(ppg, 2, 0)  # 320 PPG -> 160 * 2
         spk = np.load(spk)
 
         pit = torch.FloatTensor(pit)
+        vec = torch.FloatTensor(vec)
         ppg = torch.FloatTensor(ppg)
         spk = torch.FloatTensor(spk)
 
         len_pit = pit.size()[0]
+        len_vec = vec.size()[0] - 2 # for safe
         len_ppg = ppg.size()[0] - 2 # for safe
-        len_min = min(len_pit, len_ppg)
+        len_min = min(len_pit, len_vec)
+        len_min = min(len_min, len_ppg)
         len_wav = len_min * self.hop_length
 
         pit = pit[:len_min]
+        vec = vec[:len_min, :]
         ppg = ppg[:len_min, :]
         spe = spe[:, :len_min]
         wav = wav[:, :len_wav]
@@ -101,6 +110,7 @@ class TextAudioSpeakerSet(torch.utils.data.Dataset):
             frame_end = frame_start + use
 
             pit = pit[frame_start:frame_end]
+            vec = vec[frame_start:frame_end, :]
             ppg = ppg[frame_start:frame_end, :]
             spe = spe[:, frame_start:frame_end]
 
@@ -112,7 +122,7 @@ class TextAudioSpeakerSet(torch.utils.data.Dataset):
         # print(ppg.shape)
         # print(pit.shape)
         # print(spk.shape)
-        return spe, wav, ppg, pit, spk
+        return spe, wav, ppg, vec, pit, spk
 
 
 class TextAudioSpeakerCollate:
@@ -143,10 +153,13 @@ class TextAudioSpeakerCollate:
         ppg_lengths = torch.FloatTensor(len(batch))
         ppg_padded = torch.FloatTensor(
             len(batch), max_ppg_len, batch[0][2].size(1))
+        vec_padded = torch.FloatTensor(
+            len(batch), max_ppg_len, batch[0][3].size(1))
         pit_padded = torch.FloatTensor(len(batch), max_ppg_len)
         ppg_padded.zero_()
+        vec_padded.zero_()
         pit_padded.zero_()
-        spk = torch.FloatTensor(len(batch), batch[0][4].size(0))
+        spk = torch.FloatTensor(len(batch), batch[0][5].size(0))
 
         for i in range(len(ids_sorted_decreasing)):
             row = batch[ids_sorted_decreasing[i]]
@@ -163,10 +176,13 @@ class TextAudioSpeakerCollate:
             ppg_padded[i, : ppg.size(0), :] = ppg
             ppg_lengths[i] = ppg.size(0)
 
-            pit = row[3]
+            vec = row[3]
+            vec_padded[i, : vec.size(0), :] = vec
+
+            pit = row[4]
             pit_padded[i, : pit.size(0)] = pit
 
-            spk[i] = row[4]
+            spk[i] = row[5]
         # print(ppg_padded.shape)
         # print(ppg_lengths.shape)
         # print(pit_padded.shape)
@@ -178,6 +194,7 @@ class TextAudioSpeakerCollate:
         return (
             ppg_padded,
             ppg_lengths,
+            vec_padded,
             pit_padded,
             spk,
             spe_padded,

vits/models.py

@@ -14,6 +14,7 @@ from vits.modules_grl import SpeakerClassifier
 class TextEncoder(nn.Module):
     def __init__(self,
                  in_channels,
+                 vec_channels,
                  out_channels,
                  hidden_channels,
                  filter_channels,
@@ -24,6 +25,7 @@ class TextEncoder(nn.Module):
         super().__init__()
         self.out_channels = out_channels
         self.pre = nn.Conv1d(in_channels, hidden_channels, kernel_size=5, padding=2)
+        self.hub = nn.Conv1d(vec_channels, hidden_channels, kernel_size=5, padding=2)
         self.pit = nn.Embedding(256, hidden_channels)
         self.enc = attentions.Encoder(
             hidden_channels,
@@ -34,13 +36,15 @@ class TextEncoder(nn.Module):
             p_dropout)
         self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
 
-    def forward(self, x, x_lengths, f0):
+    def forward(self, x, x_lengths, v, f0):
         x = torch.transpose(x, 1, -1)  # [b, h, t]
         x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
             x.dtype
         )
         x = self.pre(x) * x_mask
-        x = x + self.pit(f0).transpose(1, 2)
+        v = torch.transpose(v, 1, -1)  # [b, h, t]
+        v = self.hub(v) * x_mask
+        x = x + v + self.pit(f0).transpose(1, 2)
         x = self.enc(x * x_mask, x_mask)
         stats = self.proj(x) * x_mask
         m, logs = torch.split(stats, self.out_channels, dim=1)
@@ -144,6 +148,7 @@ class SynthesizerTrn(nn.Module):
         self.emb_g = nn.Linear(hp.vits.spk_dim, hp.vits.gin_channels)
         self.enc_p = TextEncoder(
             hp.vits.ppg_dim,
+            hp.vits.vec_dim,
             hp.vits.inter_channels,
             hp.vits.hidden_channels,
             hp.vits.filter_channels,
@@ -175,11 +180,12 @@ class SynthesizerTrn(nn.Module):
         )
         self.dec = Generator(hp=hp)
 
-    def forward(self, ppg, pit, spec, spk, ppg_l, spec_l):
-        ppg = ppg + torch.randn_like(ppg)  # Perturbation
+    def forward(self, ppg, vec, pit, spec, spk, ppg_l, spec_l):
+        ppg = ppg + torch.randn_like(ppg) * 1  # Perturbation
+        vec = vec + torch.randn_like(vec) * 2  # Perturbation
         g = self.emb_g(F.normalize(spk)).unsqueeze(-1)
         z_p, m_p, logs_p, ppg_mask, x = self.enc_p(
-            ppg, ppg_l, f0=f0_to_coarse(pit))
+            ppg, ppg_l, vec, f0=f0_to_coarse(pit))
         z_q, m_q, logs_q, spec_mask = self.enc_q(spec, spec_l, g=g)
 
         z_slice, pit_slice, ids_slice = commons.rand_slice_segments_with_pitch(
@@ -193,10 +199,10 @@ class SynthesizerTrn(nn.Module):
         spk_preds = self.speaker_classifier(x)
         return audio, ids_slice, spec_mask, (z_f, z_r, z_p, m_p, logs_p, z_q, m_q, logs_q, logdet_f, logdet_r), spk_preds
 
-    def infer(self, ppg, pit, spk, ppg_l):
+    def infer(self, ppg, vec, pit, spk, ppg_l):
         ppg = ppg + torch.randn_like(ppg) * 0.0001  # Perturbation
         z_p, m_p, logs_p, ppg_mask, x = self.enc_p(
-            ppg, ppg_l, f0=f0_to_coarse(pit))
+            ppg, ppg_l, vec, f0=f0_to_coarse(pit))
         z, _ = self.flow(z_p, ppg_mask, g=spk, reverse=True)
         o = self.dec(spk, z * ppg_mask, f0=pit)
         return o
@@ -213,6 +219,7 @@ class SynthesizerInfer(nn.Module):
         self.segment_size = segment_size
         self.enc_p = TextEncoder(
             hp.vits.ppg_dim,
+            hp.vits.vec_dim,
             hp.vits.inter_channels,
             hp.vits.hidden_channels,
             hp.vits.filter_channels,
@@ -241,9 +248,9 @@ class SynthesizerInfer(nn.Module):
     def source2wav(self, source):
         return self.dec.source2wav(source)
 
-    def inference(self, ppg, pit, spk, ppg_l, source):
+    def inference(self, ppg, vec, pit, spk, ppg_l, source):
         z_p, m_p, logs_p, ppg_mask, x = self.enc_p(
-            ppg, ppg_l, f0=f0_to_coarse(pit))
+            ppg, ppg_l, vec, f0=f0_to_coarse(pit))
         z, _ = self.flow(z_p, ppg_mask, g=spk, reverse=True)
         o = self.dec.inference(spk, z * ppg_mask, source)
         return o

vits_pretrain/{sovits5.0_bigvgan.pth → sovits5.0_bigvgan_mix.pth}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffed3845044b8bef076d72272da19791e1344ad3b750a02d6e4980acf6cb0a0b
-size 74825605
+oid sha256:6b941958b20d2eb91abdb6ff9d1344e056ec2c78116e4c3a1e2b23b022d32db1
+size 79352005

whisper/inference.py

@@ -1,4 +1,5 @@
-import os
+import sys,os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import numpy as np
 import argparse
 import torch