Upload 17 files

configs/config.json

@@ -0,0 +1,90 @@
+{
+  "train": {
+    "log_interval": 200,
+    "eval_interval": 1000,
+    "seed": 1234,
+    "epochs": 10000,
+    "learning_rate": 0.0001,
+    "betas": [
+      0.8,
+      0.99
+    ],
+    "eps": 1e-09,
+    "batch_size": 12,
+    "fp16_run": false,
+    "lr_decay": 0.999875,
+    "segment_size": 17920,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0,
+    "use_sr": true,
+    "max_speclen": 384,
+    "port": "8001"
+  },
+  "data": {
+    "training_files": "filelists/train.txt",
+    "validation_files": "filelists/val.txt",
+    "max_wav_value": 32768.0,
+    "sampling_rate": 32000,
+    "filter_length": 1280,
+    "hop_length": 320,
+    "win_length": 1280,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [
+      3,
+      7,
+      11
+    ],
+    "resblock_dilation_sizes": [
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ],
+      [
+        1,
+        3,
+        5
+      ]
+    ],
+    "upsample_rates": [
+      10,
+      8,
+      2,
+      2
+    ],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [
+      16,
+      16,
+      4,
+      4
+    ],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "ssl_dim": 256,
+    "n_speakers": 2
+  },
+  "spk": {
+    "Ztech": 0
+  }
+}

filelists/test.txt

@@ -0,0 +1,7 @@
+./dataset/32k/yunhao/001829.wav
+./dataset/32k/yunhao/001827.wav
+./dataset/32k/jishuang/000104.wav
+./dataset/32k/nen/kne110_005.wav
+./dataset/32k/nen/kne110_004.wav
+./dataset/32k/jishuang/000223.wav
+./dataset/32k/yunhao/001828.wav

filelists/val.txt

@@ -0,0 +1,6 @@
+./dataset/32k/nen/kne110_005.wav
+./dataset/32k/yunhao/001827.wav
+./dataset/32k/jishuang/000104.wav
+./dataset/32k/jishuang/000223.wav
+./dataset/32k/nen/kne110_004.wav
+./dataset/32k/yunhao/001828.wav

hubert/app.py

@@ -0,0 +1,70 @@
+import io
+
+import gradio as gr
+import librosa
+import numpy as np
+import soundfile
+import torch
+from inference.infer_tool import Svc
+import logging
+
+logging.getLogger('numba').setLevel(logging.WARNING)
+
+model_name = "logs/32k/G_98000.pth"
+config_name = "configs/config.json"
+
+svc_model = Svc(model_name, config_name)
+sid_map = {
+    "Ztech": "Ztech"
+}
+
+
+def vc_fn(sid, input_audio, vc_transform):
+    if input_audio is None:
+        return "You need to upload an audio", None
+    sampling_rate, audio = input_audio
+    # print(audio.shape,sampling_rate)
+    duration = audio.shape[0] / sampling_rate
+    if duration > 45:
+        return "请上传小于45s的音频，需要转换长音频请本地进行转换", None
+    audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+    if len(audio.shape) > 1:
+        audio = librosa.to_mono(audio.transpose(1, 0))
+    if sampling_rate != 16000:
+        audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
+    print(audio.shape)
+    out_wav_path = io.BytesIO()
+    soundfile.write(out_wav_path, audio, 16000, format="wav")
+    out_wav_path.seek(0)
+
+    sid = sid_map[sid]
+    out_audio, out_sr = svc_model.infer(sid, vc_transform, out_wav_path)
+    _audio = out_audio.cpu().numpy()
+    return "Success", (32000, _audio)
+
+
+app = gr.Blocks()
+with app:
+    with gr.Tabs():
+        with gr.TabItem("Basic"):
+            gr.Markdown(value="""
+                这是sovits 3.0 32khz版本ai草莓猫taffy的在线demo
+                
+                在使用此模型前请阅读[AI粘连科技模型使用协议](https://huggingface.co/spaces/reha/Stick_Tech/blob/main/terms.md)
+                 
+               粘连科技Official@bilibili：[点击关注](https://space.bilibili.com/248582596)
+                
+                如果要在本地使用该demo，请使用git lfs clone 该仓库，安装requirements.txt后运行app.py即可
+                
+                项目改写基于 https://huggingface.co/spaces/innnky/nyaru-svc-3.0
+                
+                本地合成可以删除26、27两行代码以解除合成45s长度限制""")
+            sid = gr.Dropdown(label="音色", choices=["taffy"], value="taffy")
+            vc_input3 = gr.Audio(label="上传音频（长度小于45秒）")
+            vc_transform = gr.Number(label="变调（整数，可以正负，半音数量，升高八度就是12）", value=0)
+            vc_submit = gr.Button("转换", variant="primary")
+            vc_output1 = gr.Textbox(label="Output Message")
+            vc_output2 = gr.Audio(label="Output Audio")
+        vc_submit.click(vc_fn, [sid, vc_input3, vc_transform], [vc_output1, vc_output2])
+
+    app.launch()

hubert/hubert-soft-0d54a1f4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82e7d079df05fe3aa535f6f7d42d309bdae1d2a53324e2b2386c56721f4f649
+size 378435957

hubert/hubert_model.py

@@ -0,0 +1,222 @@
+import copy
+import random
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as t_func
+from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
+
+
+class Hubert(nn.Module):
+    def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
+        super().__init__()
+        self._mask = mask
+        self.feature_extractor = FeatureExtractor()
+        self.feature_projection = FeatureProjection()
+        self.positional_embedding = PositionalConvEmbedding()
+        self.norm = nn.LayerNorm(768)
+        self.dropout = nn.Dropout(0.1)
+        self.encoder = TransformerEncoder(
+            nn.TransformerEncoderLayer(
+                768, 12, 3072, activation="gelu", batch_first=True
+            ),
+            12,
+        )
+        self.proj = nn.Linear(768, 256)
+
+        self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
+        self.label_embedding = nn.Embedding(num_label_embeddings, 256)
+
+    def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        mask = None
+        if self.training and self._mask:
+            mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
+            x[mask] = self.masked_spec_embed.to(x.dtype)
+        return x, mask
+
+    def encode(
+            self, x: torch.Tensor, layer: Optional[int] = None
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        x = self.feature_extractor(x)
+        x = self.feature_projection(x.transpose(1, 2))
+        x, mask = self.mask(x)
+        x = x + self.positional_embedding(x)
+        x = self.dropout(self.norm(x))
+        x = self.encoder(x, output_layer=layer)
+        return x, mask
+
+    def logits(self, x: torch.Tensor) -> torch.Tensor:
+        logits = torch.cosine_similarity(
+            x.unsqueeze(2),
+            self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
+            dim=-1,
+        )
+        return logits / 0.1
+
+    def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        x, mask = self.encode(x)
+        x = self.proj(x)
+        logits = self.logits(x)
+        return logits, mask
+
+
+class HubertSoft(Hubert):
+    def __init__(self):
+        super().__init__()
+
+    @torch.inference_mode()
+    def units(self, wav: torch.Tensor) -> torch.Tensor:
+        wav = t_func.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
+        x, _ = self.encode(wav)
+        return self.proj(x)
+
+
+class FeatureExtractor(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
+        self.norm0 = nn.GroupNorm(512, 512)
+        self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
+        self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
+        self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = t_func.gelu(self.norm0(self.conv0(x)))
+        x = t_func.gelu(self.conv1(x))
+        x = t_func.gelu(self.conv2(x))
+        x = t_func.gelu(self.conv3(x))
+        x = t_func.gelu(self.conv4(x))
+        x = t_func.gelu(self.conv5(x))
+        x = t_func.gelu(self.conv6(x))
+        return x
+
+
+class FeatureProjection(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.norm = nn.LayerNorm(512)
+        self.projection = nn.Linear(512, 768)
+        self.dropout = nn.Dropout(0.1)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.norm(x)
+        x = self.projection(x)
+        x = self.dropout(x)
+        return x
+
+
+class PositionalConvEmbedding(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.conv = nn.Conv1d(
+            768,
+            768,
+            kernel_size=128,
+            padding=128 // 2,
+            groups=16,
+        )
+        self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.conv(x.transpose(1, 2))
+        x = t_func.gelu(x[:, :, :-1])
+        return x.transpose(1, 2)
+
+
+class TransformerEncoder(nn.Module):
+    def __init__(
+            self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
+    ) -> None:
+        super(TransformerEncoder, self).__init__()
+        self.layers = nn.ModuleList(
+            [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
+        )
+        self.num_layers = num_layers
+
+    def forward(
+            self,
+            src: torch.Tensor,
+            mask: torch.Tensor = None,
+            src_key_padding_mask: torch.Tensor = None,
+            output_layer: Optional[int] = None,
+    ) -> torch.Tensor:
+        output = src
+        for layer in self.layers[:output_layer]:
+            output = layer(
+                output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
+            )
+        return output
+
+
+def _compute_mask(
+        shape: Tuple[int, int],
+        mask_prob: float,
+        mask_length: int,
+        device: torch.device,
+        min_masks: int = 0,
+) -> torch.Tensor:
+    batch_size, sequence_length = shape
+
+    if mask_length < 1:
+        raise ValueError("`mask_length` has to be bigger than 0.")
+
+    if mask_length > sequence_length:
+        raise ValueError(
+            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
+        )
+
+    # compute number of masked spans in batch
+    num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
+    num_masked_spans = max(num_masked_spans, min_masks)
+
+    # make sure num masked indices <= sequence_length
+    if num_masked_spans * mask_length > sequence_length:
+        num_masked_spans = sequence_length // mask_length
+
+    # SpecAugment mask to fill
+    mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
+
+    # uniform distribution to sample from, make sure that offset samples are < sequence_length
+    uniform_dist = torch.ones(
+        (batch_size, sequence_length - (mask_length - 1)), device=device
+    )
+
+    # get random indices to mask
+    mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
+
+    # expand masked indices to masked spans
+    mask_indices = (
+        mask_indices.unsqueeze(dim=-1)
+        .expand((batch_size, num_masked_spans, mask_length))
+        .reshape(batch_size, num_masked_spans * mask_length)
+    )
+    offsets = (
+        torch.arange(mask_length, device=device)[None, None, :]
+        .expand((batch_size, num_masked_spans, mask_length))
+        .reshape(batch_size, num_masked_spans * mask_length)
+    )
+    mask_idxs = mask_indices + offsets
+
+    # scatter indices to mask
+    mask = mask.scatter(1, mask_idxs, True)
+
+    return mask
+
+
+def hubert_soft(
+        path: str,
+) -> HubertSoft:
+    r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
+    Args:
+        path (str): path of a pretrained model
+    """
+    hubert = HubertSoft()
+    checkpoint = torch.load(path)
+    consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+    hubert.load_state_dict(checkpoint)
+    hubert.eval()
+    return hubert

inference/chunks_temp.json

@@ -0,0 +1 @@
+{"info": "temp_dict", "cd65e3ce661250b7aea16ea398d13925": {"chunks": {"0": {"slice": false, "split_time": "0,556685"}}, "time": 1670798600}, "2959d9aae0e4172f27b54e452bf3b77c": {"chunks": {"0": {"slice": false, "split_time": "0,298726"}, "1": {"slice": true, "split_time": "298726,303854"}, "2": {"slice": false, "split_time": "303854,631469"}}, "time": 1670427122}, "c617b9cc74eeed7940d9e6f47c0c5bb6": {"chunks": {"0": {"slice": true, "split_time": "0,294190"}, "1": {"slice": false, "split_time": "294190,2014632"}, "2": {"slice": true, "split_time": "2014632,2021279"}, "3": {"slice": false, "split_time": "2021279,2800739"}, "4": {"slice": true, "split_time": "2800739,2816932"}, "5": {"slice": false, "split_time": "2816932,4554807"}, "6": {"slice": true, "split_time": "4554807,4572392"}, "7": {"slice": false, "split_time": "4572392,5337074"}, "8": {"slice": true, "split_time": "5337074,6197945"}, "9": {"slice": false, "split_time": "6197945,7043120"}, "10": {"slice": true, "split_time": "7043120,7195239"}}, "time": 1670428761}, "e7bba7ef02e7bba00520d7171a529b02": {"chunks": {"0": {"slice": false, "split_time": "0,435739"}}, "time": 1670472899}, "3517fa06b9fe06618393107005ce145f": {"chunks": {"0": {"slice": false, "split_time": "0,245893"}}, "time": 1670475779}, "2250dc696d4c0025d766e5234912e446": {"chunks": {"0": {"slice": true, "split_time": "0,223394"}, "1": {"slice": false, "split_time": "223394,546311"}, "2": {"slice": true, "split_time": "546311,572526"}, "3": {"slice": false, "split_time": "572526,1004349"}, "4": {"slice": true, "split_time": "1004349,1090615"}, "5": {"slice": false, "split_time": "1090615,1415280"}, "6": {"slice": true, "split_time": "1415280,1418069"}, "7": {"slice": false, "split_time": "1418069,1659131"}, "8": {"slice": true, "split_time": "1659131,1661453"}, "9": {"slice": false, "split_time": "1661453,1888827"}, "10": {"slice": true, "split_time": "1888827,1960051"}, "11": {"slice": false, "split_time": "1960051,2230836"}, "12": {"slice": true, "split_time": "2230836,2306854"}, "13": {"slice": false, "split_time": "2306854,2583422"}, "14": {"slice": true, "split_time": "2583422,2649271"}, "15": {"slice": false, "split_time": "2649271,2929916"}, "16": {"slice": true, "split_time": "2929916,2977116"}, "17": {"slice": false, "split_time": "2977116,3431901"}, "18": {"slice": true, "split_time": "3431901,3504853"}}, "time": 1670476034}, "7030c457119b4710d0091ce67f58a125": {"chunks": {"0": {"slice": true, "split_time": "0,9640"}, "1": {"slice": false, "split_time": "9640,209081"}, "2": {"slice": true, "split_time": "209081,210126"}, "3": {"slice": false, "split_time": "210126,504084"}, "4": {"slice": true, "split_time": "504084,505625"}, "5": {"slice": false, "split_time": "505625,768061"}, "6": {"slice": true, "split_time": "768061,795550"}}, "time": 1670627906}, "7d9274b960035df4dacfdc95b492cf7c": {"chunks": {"0": {"slice": false, "split_time": "0,337196"}, "1": {"slice": true, "split_time": "337196,347378"}, "2": {"slice": false, "split_time": "347378,1022501"}, "3": {"slice": true, "split_time": "1022501,1034918"}, "4": {"slice": false, "split_time": "1034918,2070080"}}, "time": 1670487808}, "0f3c73ebbda2101325eb6453551514af": {"chunks": {"0": {"slice": true, "split_time": "0,475043"}, "1": {"slice": false, "split_time": "475043,1288182"}, "2": {"slice": true, "split_time": "1288182,1303033"}, "3": {"slice": false, "split_time": "1303033,2101474"}, "4": {"slice": true, "split_time": "2101474,2106811"}, "5": {"slice": false, "split_time": "2106811,3055223"}, "6": {"slice": true, "split_time": "3055223,3516745"}, "7": {"slice": false, "split_time": "3516745,4348812"}, "8": {"slice": true, "split_time": "4348812,4354034"}, "9": {"slice": false, "split_time": "4354034,4756434"}, "10": {"slice": true, "split_time": "4756434,4757558"}, "11": {"slice": false, "split_time": "4757558,7830503"}, "12": {"slice": true, "split_time": "7830503,7839320"}, "13": {"slice": false, "split_time": "7839320,8051918"}, "14": {"slice": true, "split_time": "8051918,8184993"}}, "time": 1670713815}, "c544d7842bb2fa325a0aa9a21b7f9503": {"chunks": {"0": {"slice": true, "split_time": "0,134375"}, "1": {"slice": false, "split_time": "134375,947514"}, "2": {"slice": true, "split_time": "947514,962365"}, "3": {"slice": false, "split_time": "962365,1760806"}, "4": {"slice": true, "split_time": "1760806,1766143"}, "5": {"slice": false, "split_time": "1766143,2714555"}, "6": {"slice": true, "split_time": "2714555,2888274"}}, "time": 1670586817}, "b663f58fd3b1da710febfa0a45f447f7": {"chunks": {"0": {"slice": false, "split_time": "0,333120"}}, "time": 1670628277}, "dccf823870a0c278d53690469e26ce5e": {"chunks": {"0": {"slice": false, "split_time": "0,174322"}, "1": {"slice": true, "split_time": "174322,178984"}, "2": {"slice": false, "split_time": "178984,592293"}}, "time": 1670750467}, "1ea41e9dc88c7c28ac2ac7fc637f929f": {"chunks": {"0": {"slice": false, "split_time": "0,177025"}, "1": {"slice": true, "split_time": "177025,180753"}, "2": {"slice": false, "split_time": "180753,307984"}, "3": {"slice": true, "split_time": "307984,309679"}, "4": {"slice": false, "split_time": "309679,602116"}, "5": {"slice": true, "split_time": "602116,604118"}, "6": {"slice": false, "split_time": "604118,608816"}}, "time": 1670750647}, "70bfe94e1cfe1b9eb8e575d15a7e8bb2": {"chunks": {"0": {"slice": false, "split_time": "0,240584"}}, "time": 1670751256}, "a4296c8efc33bd7857cb3b2f6f9b912b": {"chunks": {"0": {"slice": false, "split_time": "0,314870"}}, "time": 1670751262}, "4f0e18a681b221edfb66f601cb0257dc": {"chunks": {"0": {"slice": true, "split_time": "0,13985"}, "1": {"slice": false, "split_time": "13985,114094"}, "2": {"slice": true, "split_time": "114094,117459"}, "3": {"slice": false, "split_time": "117459,326432"}, "4": {"slice": true, "split_time": "326432,328157"}, "5": {"slice": false, "split_time": "328157,462942"}, "6": {"slice": true, "split_time": "462942,480279"}, "7": {"slice": false, "split_time": "480279,615909"}, "8": {"slice": true, "split_time": "615909,620472"}}, "time": 1670751397}}

inference/infer_tool.py

@@ -0,0 +1,326 @@
+import hashlib
+import json
+import logging
+import os
+import time
+from pathlib import Path
+
+import librosa
+import maad
+import numpy as np
+# import onnxruntime
+import parselmouth
+import soundfile
+import torch
+import torchaudio
+
+from hubert import hubert_model
+import utils
+from models import SynthesizerTrn
+
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+
+
+def read_temp(file_name):
+    if not os.path.exists(file_name):
+        with open(file_name, "w") as f:
+            f.write(json.dumps({"info": "temp_dict"}))
+        return {}
+    else:
+        try:
+            with open(file_name, "r") as f:
+                data = f.read()
+            data_dict = json.loads(data)
+            if os.path.getsize(file_name) > 50 * 1024 * 1024:
+                f_name = file_name.split("/")[-1]
+                print(f"clean {f_name}")
+                for wav_hash in list(data_dict.keys()):
+                    if int(time.time()) - int(data_dict[wav_hash]["time"]) > 14 * 24 * 3600:
+                        del data_dict[wav_hash]
+        except Exception as e:
+            print(e)
+            print(f"{file_name} error,auto rebuild file")
+            data_dict = {"info": "temp_dict"}
+        return data_dict
+
+
+def write_temp(file_name, data):
+    with open(file_name, "w") as f:
+        f.write(json.dumps(data))
+
+
+def timeit(func):
+    def run(*args, **kwargs):
+        t = time.time()
+        res = func(*args, **kwargs)
+        print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
+        return res
+
+    return run
+
+
+def format_wav(audio_path):
+    if Path(audio_path).suffix == '.wav':
+        return
+    raw_audio, raw_sample_rate = librosa.load(audio_path, mono=True, sr=None)
+    soundfile.write(Path(audio_path).with_suffix(".wav"), raw_audio, raw_sample_rate)
+
+
+def get_end_file(dir_path, end):
+    file_lists = []
+    for root, dirs, files in os.walk(dir_path):
+        files = [f for f in files if f[0] != '.']
+        dirs[:] = [d for d in dirs if d[0] != '.']
+        for f_file in files:
+            if f_file.endswith(end):
+                file_lists.append(os.path.join(root, f_file).replace("\\", "/"))
+    return file_lists
+
+
+def get_md5(content):
+    return hashlib.new("md5", content).hexdigest()
+
+
+def resize2d_f0(x, target_len):
+    source = np.array(x)
+    source[source < 0.001] = np.nan
+    target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
+                       source)
+    res = np.nan_to_num(target)
+    return res
+
+def get_f0(x, p_len,f0_up_key=0):
+
+    time_step = 160 / 16000 * 1000
+    f0_min = 50
+    f0_max = 1100
+    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+    f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
+        time_step=time_step / 1000, voicing_threshold=0.6,
+        pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
+
+    pad_size=(p_len - len(f0) + 1) // 2
+    if(pad_size>0 or p_len - len(f0) - pad_size>0):
+        f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
+
+    f0 *= pow(2, f0_up_key / 12)
+    f0_mel = 1127 * np.log(1 + f0 / 700)
+    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
+    f0_mel[f0_mel <= 1] = 1
+    f0_mel[f0_mel > 255] = 255
+    f0_coarse = np.rint(f0_mel).astype(np.int)
+    return f0_coarse, f0
+
+def clean_pitch(input_pitch):
+    num_nan = np.sum(input_pitch == 1)
+    if num_nan / len(input_pitch) > 0.9:
+        input_pitch[input_pitch != 1] = 1
+    return input_pitch
+
+
+def plt_pitch(input_pitch):
+    input_pitch = input_pitch.astype(float)
+    input_pitch[input_pitch == 1] = np.nan
+    return input_pitch
+
+
+def f0_to_pitch(ff):
+    f0_pitch = 69 + 12 * np.log2(ff / 440)
+    return f0_pitch
+
+
+def fill_a_to_b(a, b):
+    if len(a) < len(b):
+        for _ in range(0, len(b) - len(a)):
+            a.append(a[0])
+
+
+def mkdir(paths: list):
+    for path in paths:
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+
+class Svc(object):
+    def __init__(self, net_g_path, config_path, hubert_path="hubert/hubert-soft-0d54a1f4.pt",
+                 onnx=False):
+        self.onnx = onnx
+        self.net_g_path = net_g_path
+        self.hubert_path = hubert_path
+        self.dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.net_g_ms = None
+        self.hps_ms = utils.get_hparams_from_file(config_path)
+        self.target_sample = self.hps_ms.data.sampling_rate
+        self.hop_size = self.hps_ms.data.hop_length
+        self.speakers = {}
+        for spk, sid in self.hps_ms.spk.items():
+            self.speakers[sid] = spk
+        self.spk2id = self.hps_ms.spk
+        # 加载hubert
+        self.hubert_soft = hubert_model.hubert_soft(hubert_path)
+        if torch.cuda.is_available():
+            self.hubert_soft = self.hubert_soft.cuda()
+        self.load_model()
+
+    def load_model(self):
+        # 获取模型配置
+        if self.onnx:
+            raise NotImplementedError
+            # self.net_g_ms = SynthesizerTrnForONNX(
+            #     178,
+            #     self.hps_ms.data.filter_length // 2 + 1,
+            #     self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
+            #     n_speakers=self.hps_ms.data.n_speakers,
+            #     **self.hps_ms.model)
+            # _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
+        else:
+            self.net_g_ms = SynthesizerTrn(
+                self.hps_ms.data.filter_length // 2 + 1,
+                self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
+                **self.hps_ms.model)
+            _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
+        if "half" in self.net_g_path and torch.cuda.is_available():
+            _ = self.net_g_ms.half().eval().to(self.dev)
+        else:
+            _ = self.net_g_ms.eval().to(self.dev)
+
+    def get_units(self, source, sr):
+
+        source = source.unsqueeze(0).to(self.dev)
+        with torch.inference_mode():
+            start = time.time()
+            units = self.hubert_soft.units(source)
+            use_time = time.time() - start
+            print("hubert use time:{}".format(use_time))
+            return units
+
+
+    def get_unit_pitch(self, in_path, tran):
+        source, sr = torchaudio.load(in_path)
+        source = torchaudio.functional.resample(source, sr, 16000)
+        if len(source.shape) == 2 and source.shape[1] >= 2:
+            source = torch.mean(source, dim=0).unsqueeze(0)
+        soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
+        f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran)
+        return soft, f0
+
+    def infer(self, speaker_id, tran, raw_path):
+        if type(speaker_id) == str:
+            speaker_id = self.spk2id[speaker_id]
+        sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
+        soft, pitch = self.get_unit_pitch(raw_path, tran)
+        f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.dev)
+        if "half" in self.net_g_path and torch.cuda.is_available():
+            stn_tst = torch.HalfTensor(soft)
+        else:
+            stn_tst = torch.FloatTensor(soft)
+        with torch.no_grad():
+            x_tst = stn_tst.unsqueeze(0).to(self.dev)
+            start = time.time()
+            x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
+            audio = self.net_g_ms.infer(x_tst, f0=f0, g=sid)[0,0].data.float()
+            use_time = time.time() - start
+            print("vits use time:{}".format(use_time))
+        return audio, audio.shape[-1]
+
+
+# class SvcONNXInferModel(object):
+#     def __init__(self, hubert_onnx, vits_onnx, config_path):
+#         self.config_path = config_path
+#         self.vits_onnx = vits_onnx
+#         self.hubert_onnx = hubert_onnx
+#         self.hubert_onnx_session = onnxruntime.InferenceSession(hubert_onnx, providers=['CUDAExecutionProvider', ])
+#         self.inspect_onnx(self.hubert_onnx_session)
+#         self.vits_onnx_session = onnxruntime.InferenceSession(vits_onnx, providers=['CUDAExecutionProvider', ])
+#         self.inspect_onnx(self.vits_onnx_session)
+#         self.hps_ms = utils.get_hparams_from_file(self.config_path)
+#         self.target_sample = self.hps_ms.data.sampling_rate
+#         self.feature_input = FeatureInput(self.hps_ms.data.sampling_rate, self.hps_ms.data.hop_length)
+#
+#     @staticmethod
+#     def inspect_onnx(session):
+#         for i in session.get_inputs():
+#             print("name:{}\tshape:{}\tdtype:{}".format(i.name, i.shape, i.type))
+#         for i in session.get_outputs():
+#             print("name:{}\tshape:{}\tdtype:{}".format(i.name, i.shape, i.type))
+#
+#     def infer(self, speaker_id, tran, raw_path):
+#         sid = np.array([int(speaker_id)], dtype=np.int64)
+#         soft, pitch = self.get_unit_pitch(raw_path, tran)
+#         pitch = np.expand_dims(pitch, axis=0).astype(np.int64)
+#         stn_tst = soft
+#         x_tst = np.expand_dims(stn_tst, axis=0)
+#         x_tst_lengths = np.array([stn_tst.shape[0]], dtype=np.int64)
+#         # 使用ONNX Runtime进行推理
+#         start = time.time()
+#         audio = self.vits_onnx_session.run(output_names=["audio"],
+#                                            input_feed={
+#                                                "hidden_unit": x_tst,
+#                                                "lengths": x_tst_lengths,
+#                                                "pitch": pitch,
+#                                                "sid": sid,
+#                                            })[0][0, 0]
+#         use_time = time.time() - start
+#         print("vits_onnx_session.run time:{}".format(use_time))
+#         audio = torch.from_numpy(audio)
+#         return audio, audio.shape[-1]
+#
+#     def get_units(self, source, sr):
+#         source = torchaudio.functional.resample(source, sr, 16000)
+#         if len(source.shape) == 2 and source.shape[1] >= 2:
+#             source = torch.mean(source, dim=0).unsqueeze(0)
+#         source = source.unsqueeze(0)
+#         # 使用ONNX Runtime进行推理
+#         start = time.time()
+#         units = self.hubert_onnx_session.run(output_names=["embed"],
+#                                              input_feed={"source": source.numpy()})[0]
+#         use_time = time.time() - start
+#         print("hubert_onnx_session.run time:{}".format(use_time))
+#         return units
+#
+#     def transcribe(self, source, sr, length, transform):
+#         feature_pit = self.feature_input.compute_f0(source, sr)
+#         feature_pit = feature_pit * 2 ** (transform / 12)
+#         feature_pit = resize2d_f0(feature_pit, length)
+#         coarse_pit = self.feature_input.coarse_f0(feature_pit)
+#         return coarse_pit
+#
+#     def get_unit_pitch(self, in_path, tran):
+#         source, sr = torchaudio.load(in_path)
+#         soft = self.get_units(source, sr).squeeze(0)
+#         input_pitch = self.transcribe(source.numpy()[0], sr, soft.shape[0], tran)
+#         return soft, input_pitch
+
+
+class RealTimeVC:
+    def __init__(self):
+        self.last_chunk = None
+        self.last_o = None
+        self.chunk_len = 16000  # 区块长度
+        self.pre_len = 3840  # 交叉淡化长度，640的倍数
+
+    """输入输出都是1维numpy 音频波形数组"""
+
+    def process(self, svc_model, speaker_id, f_pitch_change, input_wav_path):
+        audio, sr = torchaudio.load(input_wav_path)
+        audio = audio.cpu().numpy()[0]
+        temp_wav = io.BytesIO()
+        if self.last_chunk is None:
+            input_wav_path.seek(0)
+            audio, sr = svc_model.infer(speaker_id, f_pitch_change, input_wav_path)
+            audio = audio.cpu().numpy()
+            self.last_chunk = audio[-self.pre_len:]
+            self.last_o = audio
+            return audio[-self.chunk_len:]
+        else:
+            audio = np.concatenate([self.last_chunk, audio])
+            soundfile.write(temp_wav, audio, sr, format="wav")
+            temp_wav.seek(0)
+            audio, sr = svc_model.infer(speaker_id, f_pitch_change, temp_wav)
+            audio = audio.cpu().numpy()
+            ret = maad.util.crossfade(self.last_o, audio, self.pre_len)
+            self.last_chunk = audio[-self.pre_len:]
+            self.last_o = audio
+            return ret[self.chunk_len:2 * self.chunk_len]

inference/slicer.py

@@ -0,0 +1,158 @@
+import time
+
+import numpy as np
+import torch
+import torchaudio
+from scipy.ndimage import maximum_filter1d, uniform_filter1d
+
+
+def timeit(func):
+    def run(*args, **kwargs):
+        t = time.time()
+        res = func(*args, **kwargs)
+        print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
+        return res
+
+    return run
+
+
+# @timeit
+def _window_maximum(arr, win_sz):
+    return maximum_filter1d(arr, size=win_sz)[win_sz // 2: win_sz // 2 + arr.shape[0] - win_sz + 1]
+
+
+# @timeit
+def _window_rms(arr, win_sz):
+    filtered = np.sqrt(uniform_filter1d(np.power(arr, 2), win_sz) - np.power(uniform_filter1d(arr, win_sz), 2))
+    return filtered[win_sz // 2: win_sz // 2 + arr.shape[0] - win_sz + 1]
+
+
+def level2db(levels, eps=1e-12):
+    return 20 * np.log10(np.clip(levels, a_min=eps, a_max=1))
+
+
+def _apply_slice(audio, begin, end):
+    if len(audio.shape) > 1:
+        return audio[:, begin: end]
+    else:
+        return audio[begin: end]
+
+
+class Slicer:
+    def __init__(self,
+                 sr: int,
+                 db_threshold: float = -40,
+                 min_length: int = 5000,
+                 win_l: int = 300,
+                 win_s: int = 20,
+                 max_silence_kept: int = 500):
+        self.db_threshold = db_threshold
+        self.min_samples = round(sr * min_length / 1000)
+        self.win_ln = round(sr * win_l / 1000)
+        self.win_sn = round(sr * win_s / 1000)
+        self.max_silence = round(sr * max_silence_kept / 1000)
+        if not self.min_samples >= self.win_ln >= self.win_sn:
+            raise ValueError('The following condition must be satisfied: min_length >= win_l >= win_s')
+        if not self.max_silence >= self.win_sn:
+            raise ValueError('The following condition must be satisfied: max_silence_kept >= win_s')
+
+    @timeit
+    def slice(self, audio):
+        samples = audio
+        if samples.shape[0] <= self.min_samples:
+            return {"0": {"slice": False, "split_time": f"0,{len(audio)}"}}
+        # get absolute amplitudes
+        abs_amp = np.abs(samples - np.mean(samples))
+        # calculate local maximum with large window
+        win_max_db = level2db(_window_maximum(abs_amp, win_sz=self.win_ln))
+        sil_tags = []
+        left = right = 0
+        while right < win_max_db.shape[0]:
+            if win_max_db[right] < self.db_threshold:
+                right += 1
+            elif left == right:
+                left += 1
+                right += 1
+            else:
+                if left == 0:
+                    split_loc_l = left
+                else:
+                    sil_left_n = min(self.max_silence, (right + self.win_ln - left) // 2)
+                    rms_db_left = level2db(_window_rms(samples[left: left + sil_left_n], win_sz=self.win_sn))
+                    split_win_l = left + np.argmin(rms_db_left)
+                    split_loc_l = split_win_l + np.argmin(abs_amp[split_win_l: split_win_l + self.win_sn])
+                if len(sil_tags) != 0 and split_loc_l - sil_tags[-1][1] < self.min_samples and right < win_max_db.shape[
+                    0] - 1:
+                    right += 1
+                    left = right
+                    continue
+                if right == win_max_db.shape[0] - 1:
+                    split_loc_r = right + self.win_ln
+                else:
+                    sil_right_n = min(self.max_silence, (right + self.win_ln - left) // 2)
+                    rms_db_right = level2db(_window_rms(samples[right + self.win_ln - sil_right_n: right + self.win_ln],
+                                                        win_sz=self.win_sn))
+                    split_win_r = right + self.win_ln - sil_right_n + np.argmin(rms_db_right)
+                    split_loc_r = split_win_r + np.argmin(abs_amp[split_win_r: split_win_r + self.win_sn])
+                sil_tags.append((split_loc_l, split_loc_r))
+                right += 1
+                left = right
+        if left != right:
+            sil_left_n = min(self.max_silence, (right + self.win_ln - left) // 2)
+            rms_db_left = level2db(_window_rms(samples[left: left + sil_left_n], win_sz=self.win_sn))
+            split_win_l = left + np.argmin(rms_db_left)
+            split_loc_l = split_win_l + np.argmin(abs_amp[split_win_l: split_win_l + self.win_sn])
+            sil_tags.append((split_loc_l, samples.shape[0]))
+        if len(sil_tags) == 0:
+            return {"0": {"slice": False, "split_time": f"0,{len(audio)}"}}
+        else:
+            chunks = []
+            # 第一段静音并非从头开始，补上有声片段
+            if sil_tags[0][0]:
+                chunks.append({"slice": False, "split_time": f"0,{sil_tags[0][0]}"})
+            for i in range(0, len(sil_tags)):
+                # 标识有声片段（跳过第一段）
+                if i:
+                    chunks.append({"slice": False, "split_time": f"{sil_tags[i - 1][1]},{sil_tags[i][0]}"})
+                # 标识所有静音片段
+                chunks.append({"slice": True, "split_time": f"{sil_tags[i][0]},{sil_tags[i][1]}"})
+            # 最后一段静音并非结尾，补上结尾片段
+            if sil_tags[-1][1] != len(audio):
+                chunks.append({"slice": False, "split_time": f"{sil_tags[-1][1]},{len(audio)}"})
+            chunk_dict = {}
+            for i in range(len(chunks)):
+                chunk_dict[str(i)] = chunks[i]
+            return chunk_dict
+
+
+def cut(audio_path, db_thresh=-30, min_len=5000, win_l=300, win_s=20, max_sil_kept=500):
+    audio, sr = torchaudio.load(audio_path)
+    if len(audio.shape) == 2 and audio.shape[1] >= 2:
+        audio = torch.mean(audio, dim=0).unsqueeze(0)
+    audio = audio.cpu().numpy()[0]
+
+    slicer = Slicer(
+        sr=sr,
+        db_threshold=db_thresh,
+        min_length=min_len,
+        win_l=win_l,
+        win_s=win_s,
+        max_silence_kept=max_sil_kept
+    )
+    chunks = slicer.slice(audio)
+    return chunks
+
+
+def chunks2audio(audio_path, chunks):
+    chunks = dict(chunks)
+    audio, sr = torchaudio.load(audio_path)
+    if len(audio.shape) == 2 and audio.shape[1] >= 2:
+        audio = torch.mean(audio, dim=0).unsqueeze(0)
+    audio = audio.cpu().numpy()[0]
+    result = []
+    for k, v in chunks.items():
+        tag = v["split_time"].split(",")
+        result.append((v["slice"], audio[int(tag[0]):int(tag[1])]))
+    return result, sr
+
+