Upload infer_tool_grad.py

inference/infer_tool_grad.py

@@ -0,0 +1,160 @@
+import hashlib
+import json
+import logging
+import os
+import time
+from pathlib import Path
+import io
+import librosa
+import maad
+import numpy as np
+from inference import slicer
+import parselmouth
+import soundfile
+import torch
+import torchaudio
+
+from hubert import hubert_model
+import utils
+from models import SynthesizerTrn
+logging.getLogger('numba').setLevel(logging.WARNING)
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+
+def resize2d_f0(x, target_len):
+    source = np.array(x)
+    source[source < 0.001] = np.nan
+    target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
+                       source)
+    res = np.nan_to_num(target)
+    return res
+
+def get_f0(x, p_len,f0_up_key=0):
+
+    time_step = 160 / 16000 * 1000
+    f0_min = 50
+    f0_max = 1100
+    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+    f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
+        time_step=time_step / 1000, voicing_threshold=0.6,
+        pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
+
+    pad_size=(p_len - len(f0) + 1) // 2
+    if(pad_size>0 or p_len - len(f0) - pad_size>0):
+        f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
+
+    f0 *= pow(2, f0_up_key / 12)
+    f0_mel = 1127 * np.log(1 + f0 / 700)
+    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
+    f0_mel[f0_mel <= 1] = 1
+    f0_mel[f0_mel > 255] = 255
+    f0_coarse = np.rint(f0_mel).astype(np.int)
+    return f0_coarse, f0
+
+def clean_pitch(input_pitch):
+    num_nan = np.sum(input_pitch == 1)
+    if num_nan / len(input_pitch) > 0.9:
+        input_pitch[input_pitch != 1] = 1
+    return input_pitch
+
+
+def plt_pitch(input_pitch):
+    input_pitch = input_pitch.astype(float)
+    input_pitch[input_pitch == 1] = np.nan
+    return input_pitch
+
+
+def f0_to_pitch(ff):
+    f0_pitch = 69 + 12 * np.log2(ff / 440)
+    return f0_pitch
+
+
+def fill_a_to_b(a, b):
+    if len(a) < len(b):
+        for _ in range(0, len(b) - len(a)):
+            a.append(a[0])
+
+
+def mkdir(paths: list):
+    for path in paths:
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+
+class VitsSvc(object):
+    def __init__(self):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.SVCVITS = None
+        self.hps = None
+        self.speakers = None
+        self.hubert_soft = hubert_model.hubert_soft("hubert/model.pt")
+
+    def set_device(self, device):
+        self.device = torch.device(device)
+        self.hubert_soft.to(self.device)
+        if self.SVCVITS != None:
+            self.SVCVITS.to(self.device)
+
+    def loadCheckpoint(self, path):
+        self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
+        self.SVCVITS = SynthesizerTrn(
+            self.hps.data.filter_length // 2 + 1,
+            self.hps.train.segment_size // self.hps.data.hop_length,
+            **self.hps.model)
+        _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None)
+        _ = self.SVCVITS.eval().to(self.device)
+        self.speakers = self.hps.spk
+
+    def get_units(self, source, sr):
+        source = source.unsqueeze(0).to(self.device)
+        with torch.inference_mode():
+            units = self.hubert_soft.units(source)
+            return units
+
+
+    def get_unit_pitch(self, in_path, tran):
+        source, sr = torchaudio.load(in_path)
+        source = torchaudio.functional.resample(source, sr, 16000)
+        if len(source.shape) == 2 and source.shape[1] >= 2:
+            source = torch.mean(source, dim=0).unsqueeze(0)
+        soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
+        f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0]*2, tran)
+        return soft, f0
+
+    def infer(self, speaker_id, tran, raw_path):
+        speaker_id = self.speakers[speaker_id]
+        sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0)
+        soft, pitch = self.get_unit_pitch(raw_path, tran)
+        f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device)
+        stn_tst = torch.FloatTensor(soft)
+        with torch.no_grad():
+            x_tst = stn_tst.unsqueeze(0).to(self.device)
+            x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
+            audio = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0,0].data.float()
+        return audio, audio.shape[-1]
+
+    def inference(self,srcaudio,chara,tran,slice_db):
+        sampling_rate, audio = srcaudio
+        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio.transpose(1, 0))
+        if sampling_rate != 16000:
+            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
+        soundfile.write("tmpwav.wav", audio, 16000, format="wav")
+        chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db)
+        audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks)
+        audio = []
+        for (slice_tag, data) in audio_data:
+            length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate))
+            raw_path = io.BytesIO()
+            soundfile.write(raw_path, data, audio_sr, format="wav")
+            raw_path.seek(0)
+            if slice_tag:
+                _audio = np.zeros(length)
+            else:
+                out_audio, out_sr = self.infer(chara, tran, raw_path)
+                _audio = out_audio.cpu().numpy()
+            audio.extend(list(_audio))
+        audio = (np.array(audio) * 32768.0).astype('int16')
+        return (self.hps.data.sampling_rate,audio)