first commit

.gitattributes

@@ -11,7 +11,7 @@
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
+# *.npy filter=lfs diff=lfs merge=lfs -text
 *.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+exp/default/g_00700000 filter=lfs diff=lfs merge=lfs -text
+Utils/JDC/bst.t7 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+flagged
+out.wav

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jingyi Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -7,6 +7,7 @@ sdk: gradio
 sdk_version: 4.22.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Utils/JDC/__init__.py

@@ -0,0 +1 @@
+

Utils/JDC/bst.t7.txt

@@ -0,0 +1 @@
+https://github.com/yl4579/HiFTNet/blob/main/Utils/JDC/bst.t7

Utils/JDC/model.py

@@ -0,0 +1,190 @@
+"""
+Implementation of model from:
+Kum et al. - "Joint Detection and Classification of Singing Voice Melody Using
+Convolutional Recurrent Neural Networks" (2019)
+Link: https://www.semanticscholar.org/paper/Joint-Detection-and-Classification-of-Singing-Voice-Kum-Nam/60a2ad4c7db43bace75805054603747fcd062c0d
+"""
+import torch
+from torch import nn
+        
+class JDCNet(nn.Module):
+    """
+    Joint Detection and Classification Network model for singing voice melody.
+    """
+    def __init__(self, num_class=722, seq_len=31, leaky_relu_slope=0.01):
+        super().__init__()
+        self.num_class = num_class
+
+        # input = (b, 1, 31, 513), b = batch size
+        self.conv_block = nn.Sequential(
+            nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3, padding=1, bias=False),  # out: (b, 64, 31, 513)
+            nn.BatchNorm2d(num_features=64),
+            nn.LeakyReLU(leaky_relu_slope, inplace=True),
+            nn.Conv2d(64, 64, 3, padding=1, bias=False),  # (b, 64, 31, 513)
+        )
+
+        # res blocks
+        self.res_block1 = ResBlock(in_channels=64, out_channels=128)  # (b, 128, 31, 128)
+        self.res_block2 = ResBlock(in_channels=128, out_channels=192)  # (b, 192, 31, 32)
+        self.res_block3 = ResBlock(in_channels=192, out_channels=256)  # (b, 256, 31, 8)
+
+        # pool block
+        self.pool_block = nn.Sequential(
+            nn.BatchNorm2d(num_features=256),
+            nn.LeakyReLU(leaky_relu_slope, inplace=True),
+            nn.MaxPool2d(kernel_size=(1, 4)),  # (b, 256, 31, 2)
+            nn.Dropout(p=0.2),
+        )
+
+        # maxpool layers (for auxiliary network inputs)
+        # in = (b, 128, 31, 513) from conv_block, out = (b, 128, 31, 2)
+        self.maxpool1 = nn.MaxPool2d(kernel_size=(1, 40))
+        # in = (b, 128, 31, 128) from res_block1, out = (b, 128, 31, 2)
+        self.maxpool2 = nn.MaxPool2d(kernel_size=(1, 20))
+        # in = (b, 128, 31, 32) from res_block2, out = (b, 128, 31, 2)
+        self.maxpool3 = nn.MaxPool2d(kernel_size=(1, 10))
+
+        # in = (b, 640, 31, 2), out = (b, 256, 31, 2)
+        self.detector_conv = nn.Sequential(
+            nn.Conv2d(640, 256, 1, bias=False),
+            nn.BatchNorm2d(256),
+            nn.LeakyReLU(leaky_relu_slope, inplace=True),
+            nn.Dropout(p=0.2),
+        )
+
+        # input: (b, 31, 512) - resized from (b, 256, 31, 2)
+        self.bilstm_classifier = nn.LSTM(
+            input_size=512, hidden_size=256,
+            batch_first=True, bidirectional=True)  # (b, 31, 512)
+
+        # input: (b, 31, 512) - resized from (b, 256, 31, 2)
+        self.bilstm_detector = nn.LSTM(
+            input_size=512, hidden_size=256,
+            batch_first=True, bidirectional=True)  # (b, 31, 512)
+
+        # input: (b * 31, 512)
+        self.classifier = nn.Linear(in_features=512, out_features=self.num_class)  # (b * 31, num_class)
+
+        # input: (b * 31, 512)
+        self.detector = nn.Linear(in_features=512, out_features=2)  # (b * 31, 2) - binary classifier
+
+        # initialize weights
+        self.apply(self.init_weights)
+
+    def get_feature_GAN(self, x):
+        seq_len = x.shape[-2]
+        x = x.float().transpose(-1, -2)
+        
+        convblock_out = self.conv_block(x)
+        
+        resblock1_out = self.res_block1(convblock_out)
+        resblock2_out = self.res_block2(resblock1_out)
+        resblock3_out = self.res_block3(resblock2_out)
+        poolblock_out = self.pool_block[0](resblock3_out)
+        poolblock_out = self.pool_block[1](poolblock_out)
+        
+        return poolblock_out.transpose(-1, -2)
+        
+    def get_feature(self, x):
+        seq_len = x.shape[-2]
+        x = x.float().transpose(-1, -2)
+        
+        convblock_out = self.conv_block(x)
+        
+        resblock1_out = self.res_block1(convblock_out)
+        resblock2_out = self.res_block2(resblock1_out)
+        resblock3_out = self.res_block3(resblock2_out)
+        poolblock_out = self.pool_block[0](resblock3_out)
+        poolblock_out = self.pool_block[1](poolblock_out)
+        
+        return self.pool_block[2](poolblock_out)
+        
+    def forward(self, x):
+        """
+        Returns:
+            classification_prediction, detection_prediction
+            sizes: (b, 31, 722), (b, 31, 2)
+        """
+        ###############################
+        # forward pass for classifier #
+        ###############################
+        seq_len = x.shape[-1]
+        x = x.float().transpose(-1, -2)
+        
+        convblock_out = self.conv_block(x)
+        
+        resblock1_out = self.res_block1(convblock_out)
+        resblock2_out = self.res_block2(resblock1_out)
+        resblock3_out = self.res_block3(resblock2_out)
+        
+        
+        poolblock_out = self.pool_block[0](resblock3_out)
+        poolblock_out = self.pool_block[1](poolblock_out)
+        GAN_feature = poolblock_out.transpose(-1, -2)
+        poolblock_out = self.pool_block[2](poolblock_out)
+        
+        # (b, 256, 31, 2) => (b, 31, 256, 2) => (b, 31, 512)
+        classifier_out = poolblock_out.permute(0, 2, 1, 3).contiguous().view((-1, seq_len, 512))
+        classifier_out, _ = self.bilstm_classifier(classifier_out)  # ignore the hidden states
+
+        classifier_out = classifier_out.contiguous().view((-1, 512))  # (b * 31, 512)
+        classifier_out = self.classifier(classifier_out)
+        classifier_out = classifier_out.view((-1, seq_len, self.num_class))  # (b, 31, num_class)
+        
+        # sizes: (b, 31, 722), (b, 31, 2)
+        # classifier output consists of predicted pitch classes per frame
+        # detector output consists of: (isvoice, notvoice) estimates per frame
+        return torch.abs(classifier_out.squeeze()), GAN_feature, poolblock_out
+
+    @staticmethod
+    def init_weights(m):
+        if isinstance(m, nn.Linear):
+            nn.init.kaiming_uniform_(m.weight)
+            if m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.Conv2d):
+            nn.init.xavier_normal_(m.weight)
+        elif isinstance(m, nn.LSTM) or isinstance(m, nn.LSTMCell):
+            for p in m.parameters():
+                if p.data is None:
+                    continue
+
+                if len(p.shape) >= 2:
+                    nn.init.orthogonal_(p.data)
+                else:
+                    nn.init.normal_(p.data)
+                    
+
+class ResBlock(nn.Module):
+    def __init__(self, in_channels: int, out_channels: int, leaky_relu_slope=0.01):
+        super().__init__()
+        self.downsample = in_channels != out_channels
+
+        # BN / LReLU / MaxPool layer before the conv layer - see Figure 1b in the paper
+        self.pre_conv = nn.Sequential(
+            nn.BatchNorm2d(num_features=in_channels),
+            nn.LeakyReLU(leaky_relu_slope, inplace=True),
+            nn.MaxPool2d(kernel_size=(1, 2)),  # apply downsampling on the y axis only
+        )
+
+        # conv layers
+        self.conv = nn.Sequential(
+            nn.Conv2d(in_channels=in_channels, out_channels=out_channels,
+                      kernel_size=3, padding=1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.LeakyReLU(leaky_relu_slope, inplace=True),
+            nn.Conv2d(out_channels, out_channels, 3, padding=1, bias=False),
+        )
+
+        # 1 x 1 convolution layer to match the feature dimensions
+        self.conv1by1 = None
+        if self.downsample:
+            self.conv1by1 = nn.Conv2d(in_channels, out_channels, 1, bias=False)
+
+    def forward(self, x):
+        x = self.pre_conv(x)
+        if self.downsample:
+            x = self.conv(x) + self.conv1by1(x)
+        else:
+            x = self.conv(x) + x
+        return x

Utils/__init__.py

@@ -0,0 +1 @@
+

app.py

@@ -0,0 +1,142 @@
+import os
+import json
+import math
+
+import torch
+import torch.nn.functional as F
+import librosa
+import numpy as np
+import soundfile as sf
+import gradio as gr
+from transformers import WavLMModel
+
+from env import AttrDict
+from meldataset import mel_spectrogram, MAX_WAV_VALUE
+from models import Generator
+from stft import TorchSTFT
+from Utils.JDC.model import JDCNet
+
+
+# files
+hpfile = "config_v1_16k.json"
+ptfile = "exp/default/g_00700000"
+spk2id_path = "filelists/spk2id.json"
+f0_stats_path = "filelists/f0_stats.json"
+spk_stats_path = "filelists/spk_stats.json"
+spk_emb_dir = "dataset/spk"
+spk_wav_dir = "dataset/audio"
+
+# device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# load config
+with open(hpfile) as f:
+    data = f.read()
+json_config = json.loads(data)
+h = AttrDict(json_config)
+
+# load models
+F0_model = JDCNet(num_class=1, seq_len=192)
+generator = Generator(h, F0_model).to(device)
+stft = TorchSTFT(filter_length=h.gen_istft_n_fft, hop_length=h.gen_istft_hop_size, win_length=h.gen_istft_n_fft).to(device)
+
+state_dict_g = torch.load(ptfile, map_location=device)
+generator.load_state_dict(state_dict_g['generator'], strict=True)
+generator.remove_weight_norm()
+_ = generator.eval()
+
+wavlm = WavLMModel.from_pretrained("microsoft/wavlm-base-plus")
+wavlm.eval()
+wavlm.to(device)
+
+# load stats
+with open(spk2id_path) as f:
+    spk2id = json.load(f)
+with open(f0_stats_path) as f:
+    f0_stats = json.load(f)
+with open(spk_stats_path) as f:
+    spk_stats = json.load(f)
+
+# tune f0
+threshold = 10
+step = (math.log(1100) - math.log(50)) / 256
+def tune_f0(initial_f0, i):
+    if i == 0:
+        return initial_f0
+    voiced = initial_f0 > threshold
+    initial_lf0 = torch.log(initial_f0)
+    lf0 = initial_lf0 + step * i
+    f0 = torch.exp(lf0)
+    f0 = torch.where(voiced, f0, initial_f0)
+    return f0
+
+# convert function
+def convert(tgt_spk, src_wav, f0_shift=0):
+    tgt_ref = spk_stats[tgt_spk]["best_spk_emb"]
+    tgt_emb = f"{spk_emb_dir}/{tgt_spk}/{tgt_ref}.npy"
+
+    with torch.no_grad():
+        # tgt
+        spk_id = spk2id[tgt_spk]
+        spk_id = torch.LongTensor([spk_id]).unsqueeze(0).to(device)
+        
+        spk_emb = np.load(tgt_emb)
+        spk_emb = torch.from_numpy(spk_emb).unsqueeze(0).to(device)
+
+        f0_mean_tgt = f0_stats[tgt_spk]["mean"]
+
+        # src
+        wav, sr = librosa.load(src_wav, sr=16000)
+        wav = torch.FloatTensor(wav).to(device)
+        mel = mel_spectrogram(wav.unsqueeze(0), h.n_fft, h.num_mels, h.sampling_rate, h.hop_size, h.win_size, h.fmin, h.fmax)
+        
+        x = wavlm(wav.unsqueeze(0)).last_hidden_state
+        x = x.transpose(1, 2) # (B, C, T)
+        x = F.pad(x, (0, mel.size(2) - x.size(2)), 'constant')
+
+        # cvt
+        f0 = generator.get_f0(mel, f0_mean_tgt)
+        f0 = tune_f0(f0, f0_shift)
+        x = generator.get_x(x, spk_emb, spk_id)
+        y = generator.infer(x, f0, stft)
+        
+        audio = y.squeeze()
+        audio = audio / torch.max(torch.abs(audio)) * 0.95
+        audio = audio * MAX_WAV_VALUE
+        audio = audio.cpu().numpy().astype('int16')
+
+        sf.write("out.wav", audio, h.sampling_rate, "PCM_16")
+
+    out_wav = "out.wav"
+    return out_wav
+
+# change spk
+def change_spk(tgt_spk):
+    tgt_ref = spk_stats[tgt_spk]["best_spk_emb"]
+    tgt_wav = f"{spk_wav_dir}/{tgt_spk}/{tgt_ref}.wav"
+    return tgt_wav
+
+# interface
+with gr.Blocks() as demo:
+    gr.Markdown("# PitchVC")
+    gr.Markdown("Gradio Demo for PitchVC. ([Github Repo](https://github.com/OlaWod/PitchVC))")
+
+    with gr.Row():
+        with gr.Column():
+            tgt_spk = gr.Dropdown(choices=spk2id.keys(), type="value", label="Target Speaker")
+            ref_audio =  gr.Audio(label="Reference Audio", type='filepath')
+            src_audio = gr.Audio(label="Source Audio", type='filepath')
+            f0_shift = gr.Slider(minimum=-30, maximum=30, value=0, step=1, label="F0 Shift")
+        with gr.Column():
+            out_audio =  gr.Audio(label="Output Audio", type='filepath')
+            submit = gr.Button(value="Submit")
+
+    tgt_spk.change(fn=change_spk, inputs=[tgt_spk], outputs=[ref_audio])
+    submit.click(convert, [tgt_spk, src_audio, f0_shift], [out_audio])
+
+    examples = gr.Examples(
+        examples=[["p225", 'dataset/audio/p226/p226_341.wav', 0], 
+                    ["p226", 'dataset/audio/p225/p225_220.wav', -5]],
+        inputs=[tgt_spk, src_audio, f0_shift])
+
+demo.launch()

config_v1_16k.json

@@ -0,0 +1,42 @@
+{
+    "F0_path": "Utils/JDC/bst.t7",
+
+    "use_aug": true,
+
+    "resblock": "1",
+    "num_gpus": 1,
+    "batch_size": 16,
+    "learning_rate": 0.0002,
+    "adam_b1": 0.8,
+    "adam_b2": 0.99,
+    "lr_decay": 0.999,
+    "seed": 1234,
+
+    "upsample_rates": [10,8],
+    "upsample_kernel_sizes": [20,16],
+    "upsample_initial_channel": 512,
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "gen_istft_n_fft": 16,
+    "gen_istft_hop_size": 4,
+
+    "segment_size": 16000,
+    "num_mels": 80,
+    "n_fft": 1024,
+    "hop_size": 320,
+    "win_size": 1024,
+
+    "sampling_rate": 16000,
+
+    "fmin": 0,
+    "fmax": 8000,
+    "fmax_for_loss": null,
+
+    "num_workers": 8,
+
+    "dist_config": {
+        "dist_backend": "nccl",
+        "dist_url": "tcp://localhost:54321",
+        "world_size": 1
+    }
+}