Upload 7 files

config.yml

@@ -0,0 +1,176 @@
+# 全局配置
+# 对于希望在同一时间使用多个配置文件的情况，例如两个GPU同时跑两个训练集：通过环境变量指定配置文件，不指定则默认为./config.yml
+
+# 拟提供通用路径配置，统一存放数据，避免数据放得很乱
+# 每个数据集与其对应的模型存放至统一路径下，后续所有的路径配置均为相对于datasetPath的路径
+# 不填或者填空则路径为相对于项目根目录的路径
+dataset_path: "Data/tamura"
+
+# 模型镜像源，默认huggingface，使用openi镜像源需指定openi_token
+mirror: ""
+openi_token: ""  # openi token
+
+# resample 音频重采样配置
+# 注意， “:” 后需要加空格
+resample:
+  # 目标重采样率
+  sampling_rate: 44100
+  # 音频文件输入路径，重采样会将该路径下所有.wav音频文件重采样
+  # 请填入相对于datasetPath的相对路径
+  in_dir: "audios/raw" # 相对于根目录的路径为 /datasetPath/in_dir
+  # 音频文件重采样后输出路径
+  out_dir: "audios/wavs"
+
+
+# preprocess_text 数据集预处理相关配置
+# 注意， “:” 后需要加空格
+preprocess_text:
+  # 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
+  transcription_path: "filelists/text.list"
+  # 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成
+  cleaned_path: ""
+  # 训练集路径
+  train_path: "filelists/train.list"
+  # 验证集路径
+  val_path: "filelists/val.list"
+  # 配置文件路径
+  config_path: "config.json"
+  # 每个speaker的验证集条数
+  val_per_spk: 4
+  # 验证集最大条数，多于的会被截断并放到训练集中
+  max_val_total: 8
+  # 是否进行数据清洗
+  clean: true
+
+
+# bert_gen 相关配置
+# 注意， “:” 后需要加空格
+bert_gen:
+  # 训练数据集配置文件路径
+  config_path: "config.json"
+  # 并行数
+  num_processes: 2
+  # 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
+  # 该选项同时决定了get_bert_feature的默认设备
+  device: "cuda"
+  # 使用多卡推理
+  use_multi_device: false
+
+# emo_gen 相关配置
+# 注意， “:” 后需要加空格
+emo_gen:
+  # 训练数据集配置文件路径
+  config_path: "config.json"
+  # 并行数
+  num_processes: 2
+  # 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
+  device: "cuda"
+
+# train 训练配置
+# 注意， “:” 后需要加空格
+train_ms:
+  env:
+    MASTER_ADDR: "localhost"
+    MASTER_PORT: 10086
+    WORLD_SIZE: 1
+    LOCAL_RANK: 0
+    RANK: 0
+    # 可以填写任意名的环境变量
+    # THE_ENV_VAR_YOU_NEED_TO_USE: "1234567"
+  # 底模设置
+  base:
+    use_base_model: false
+    repo_id: "Stardust_minus/Bert-VITS2"
+    model_image: "Bert-VITS2_2.1-Emo底模" # openi网页的模型名
+  # 训练模型存储目录：与旧版本的区别，原先数据集是存放在logs/model_name下的，现在改为统一存放在Data/你的数据集/models下
+  model: "models"
+  # 配置文件路径
+  config_path: "config.json"
+  # 训练使用的worker，不建议超过CPU核心数
+  num_workers: 16
+  # 关闭此项可以节约接近50%的磁盘空间，但是可能导致实际训练速度变慢和更高的CPU使用率。
+  spec_cache: True
+  # 保存的检查点数量，多于此数目的权重会被删除来节省空间。
+  keep_ckpts: 8
+
+
+# webui webui配置
+# 注意， “:” 后需要加空格
+webui:
+  # 推理设备
+  device: "cuda"
+  # 模型路径
+  model: "models/G_2750.pth"
+  # 配置文件路径
+  config_path: "config.json"
+  # 端口号
+  port: 7860
+  # 是否公开部署，对外网开放
+  share: false
+  # 是否开启debug模式
+  debug: false
+  # 语种识别库，可选langid, fastlid
+  language_identification_library: "langid"
+
+
+# server api配置
+# 注意， “:” 后需要加空格
+# 注意，本配置下的所有配置均为相对于根目录的路径
+server:
+  # 端口号
+  port: 5000
+  # 模型默认使用设备：但是当前并没有实现这个配置。
+  device: "cuda"
+  # 需要加载的所有模型的配置，可以填多个模型，也可以不填模型，等网页成功后手动加载模型
+  # 不加载模型的配置格式：删除默认给的两个模型配置，给models赋值 [ ]，也就是空列表。参考模型2的speakers 即 models: [ ]
+  # 注意，所有模型都必须正确配置model与config的路径，空路径会导致加载错误。
+  # 也可以不填模型，等网页加载成功后手动填写models。
+  models:
+    - # 模型的路径
+      model: ""
+      # 模型config.json的路径
+      config: ""
+      # 模型使用设备，若填写则会覆盖默认配置
+      device: "cuda"
+      # 模型默认使用的语言
+      language: "ZH"
+      # 模型人物默认参数
+      # 不必填写所有人物，不填的使用默认值
+      # 暂时不用填写，当前尚未实现按人区分配置
+      speakers:
+        - speaker: "科比"
+          sdp_ratio: 0.2
+          noise_scale: 0.6
+          noise_scale_w: 0.8
+          length_scale: 1
+        - speaker: "五条悟"
+          sdp_ratio: 0.3
+          noise_scale: 0.7
+          noise_scale_w: 0.8
+          length_scale: 0.5
+        - speaker: "安倍晋三"
+          sdp_ratio: 0.2
+          noise_scale: 0.6
+          noise_scale_w: 0.8
+          length_scale: 1.2
+    - # 模型的路径
+      model: ""
+      # 模型config.json的路径
+      config: ""
+      # 模型使用设备，若填写则会覆盖默认配置
+      device: "cpu"
+      # 模型默认使用的语言
+      language: "JP"
+      # 模型人物默认参数
+      # 不必填写所有人物，不填的使用默认值
+      speakers: [ ] # 也可以不填
+
+
+# 百度翻译开放平台 api配置
+# api接入文档 https://api.fanyi.baidu.com/doc/21
+# 请不要在github等网站公开分享你的app id 与 key
+translate:
+  # 你的APPID
+  "app_key": ""
+  # 你的密钥
+  "secret_key": ""

data_utils.py

@@ -0,0 +1,410 @@
+import os
+import random
+import torch
+import torch.utils.data
+from tqdm import tqdm
+import numpy as np
+from tools.log import logger
+import commons
+from mel_processing import spectrogram_torch, mel_spectrogram_torch
+from utils import load_wav_to_torch, load_filepaths_and_text
+from text import cleaned_text_to_sequence
+from config import config
+
+"""Multi speaker version"""
+
+
+class TextAudioSpeakerLoader(torch.utils.data.Dataset):
+    """
+    1) loads audio, speaker_id, text pairs
+    2) normalizes text and converts them to sequences of integers
+    3) computes spectrograms from audio files.
+    """
+
+    def __init__(self, audiopaths_sid_text, hparams):
+        self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
+        self.max_wav_value = hparams.max_wav_value
+        self.sampling_rate = hparams.sampling_rate
+        self.filter_length = hparams.filter_length
+        self.hop_length = hparams.hop_length
+        self.win_length = hparams.win_length
+        self.sampling_rate = hparams.sampling_rate
+        self.spk_map = hparams.spk2id
+        self.hparams = hparams
+
+        self.use_mel_spec_posterior = getattr(
+            hparams, "use_mel_posterior_encoder", False
+        )
+        if self.use_mel_spec_posterior:
+            self.n_mel_channels = getattr(hparams, "n_mel_channels", 80)
+
+        self.cleaned_text = getattr(hparams, "cleaned_text", False)
+
+        self.add_blank = hparams.add_blank
+        self.min_text_len = getattr(hparams, "min_text_len", 1)
+        self.max_text_len = getattr(hparams, "max_text_len", 384)
+
+        random.seed(1234)
+        random.shuffle(self.audiopaths_sid_text)
+        self._filter()
+
+    def _filter(self):
+        """
+        Filter text & store spec lengths
+        """
+        # Store spectrogram lengths for Bucketing
+        # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
+        # spec_length = wav_length // hop_length
+
+        audiopaths_sid_text_new = []
+        lengths = []
+        skipped = 0
+        logger.info("Init dataset...")
+        for _id, spk, language, text, phones, tone, word2ph in tqdm(
+            self.audiopaths_sid_text
+        ):
+            audiopath = f"{_id}"
+            if self.min_text_len <= len(phones) and len(phones) <= self.max_text_len:
+                phones = phones.split(" ")
+                tone = [int(i) for i in tone.split(" ")]
+                word2ph = [int(i) for i in word2ph.split(" ")]
+                audiopaths_sid_text_new.append(
+                    [audiopath, spk, language, text, phones, tone, word2ph]
+                )
+                lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
+            else:
+                skipped += 1
+        logger.info(
+            "skipped: "
+            + str(skipped)
+            + ", total: "
+            + str(len(self.audiopaths_sid_text))
+        )
+        self.audiopaths_sid_text = audiopaths_sid_text_new
+        self.lengths = lengths
+
+    def get_audio_text_speaker_pair(self, audiopath_sid_text):
+        # separate filename, speaker_id and text
+        audiopath, sid, language, text, phones, tone, word2ph = audiopath_sid_text
+
+        bert, ja_bert, en_bert, phones, tone, language = self.get_text(
+            text, word2ph, phones, tone, language, audiopath
+        )
+
+        spec, wav = self.get_audio(audiopath)
+        sid = torch.LongTensor([int(self.spk_map[sid])])
+        emo = torch.FloatTensor(np.load(audiopath.replace(".wav", ".emo.npy")))
+        return (phones, spec, wav, sid, tone, language, bert, ja_bert, en_bert, emo)
+
+    def get_audio(self, filename):
+        audio, sampling_rate = load_wav_to_torch(filename)
+        if sampling_rate != self.sampling_rate:
+            raise ValueError(
+                "{} {} SR doesn't match target {} SR".format(
+                    filename, sampling_rate, self.sampling_rate
+                )
+            )
+        audio_norm = audio / self.max_wav_value
+        audio_norm = audio_norm.unsqueeze(0)
+        spec_filename = filename.replace(".wav", ".spec.pt")
+        if self.use_mel_spec_posterior:
+            spec_filename = spec_filename.replace(".spec.pt", ".mel.pt")
+        try:
+            spec = torch.load(spec_filename)
+        except:
+            if self.use_mel_spec_posterior:
+                spec = mel_spectrogram_torch(
+                    audio_norm,
+                    self.filter_length,
+                    self.n_mel_channels,
+                    self.sampling_rate,
+                    self.hop_length,
+                    self.win_length,
+                    self.hparams.mel_fmin,
+                    self.hparams.mel_fmax,
+                    center=False,
+                )
+            else:
+                spec = spectrogram_torch(
+                    audio_norm,
+                    self.filter_length,
+                    self.sampling_rate,
+                    self.hop_length,
+                    self.win_length,
+                    center=False,
+                )
+            spec = torch.squeeze(spec, 0)
+            if config.train_ms_config.spec_cache:
+                torch.save(spec, spec_filename)
+        return spec, audio_norm
+
+    def get_text(self, text, word2ph, phone, tone, language_str, wav_path):
+        phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+        if self.add_blank:
+            phone = commons.intersperse(phone, 0)
+            tone = commons.intersperse(tone, 0)
+            language = commons.intersperse(language, 0)
+            for i in range(len(word2ph)):
+                word2ph[i] = word2ph[i] * 2
+            word2ph[0] += 1
+        bert_path = wav_path.replace(".wav", ".bert.pt")
+        try:
+            bert_ori = torch.load(bert_path)
+            assert bert_ori.shape[-1] == len(phone)
+        except Exception as e:
+            logger.warning("Bert load Failed")
+            logger.warning(e)
+
+        if language_str == "ZH":
+            bert = bert_ori
+            ja_bert = torch.zeros(1024, len(phone))
+            en_bert = torch.zeros(1024, len(phone))
+        elif language_str == "JP":
+            bert = torch.zeros(1024, len(phone))
+            ja_bert = bert_ori
+            en_bert = torch.zeros(1024, len(phone))
+        elif language_str == "EN":
+            bert = torch.zeros(1024, len(phone))
+            ja_bert = torch.zeros(1024, len(phone))
+            en_bert = bert_ori
+        phone = torch.LongTensor(phone)
+        tone = torch.LongTensor(tone)
+        language = torch.LongTensor(language)
+        return bert, ja_bert, en_bert, phone, tone, language
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def __getitem__(self, index):
+        return self.get_audio_text_speaker_pair(self.audiopaths_sid_text[index])
+
+    def __len__(self):
+        return len(self.audiopaths_sid_text)
+
+
+class TextAudioSpeakerCollate:
+    """Zero-pads model inputs and targets"""
+
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        """Collate's training batch from normalized text, audio and speaker identities
+        PARAMS
+        ------
+        batch: [text_normalized, spec_normalized, wav_normalized, sid]
+        """
+        # Right zero-pad all one-hot text sequences to max input length
+        _, ids_sorted_decreasing = torch.sort(
+            torch.LongTensor([x[1].size(1) for x in batch]), dim=0, descending=True
+        )
+
+        max_text_len = max([len(x[0]) for x in batch])
+        max_spec_len = max([x[1].size(1) for x in batch])
+        max_wav_len = max([x[2].size(1) for x in batch])
+
+        text_lengths = torch.LongTensor(len(batch))
+        spec_lengths = torch.LongTensor(len(batch))
+        wav_lengths = torch.LongTensor(len(batch))
+        sid = torch.LongTensor(len(batch))
+
+        text_padded = torch.LongTensor(len(batch), max_text_len)
+        tone_padded = torch.LongTensor(len(batch), max_text_len)
+        language_padded = torch.LongTensor(len(batch), max_text_len)
+        bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
+        ja_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
+        en_bert_padded = torch.FloatTensor(len(batch), 1024, max_text_len)
+        emo = torch.FloatTensor(len(batch), 1024)
+
+        spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
+        wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
+        text_padded.zero_()
+        tone_padded.zero_()
+        language_padded.zero_()
+        spec_padded.zero_()
+        wav_padded.zero_()
+        bert_padded.zero_()
+        ja_bert_padded.zero_()
+        en_bert_padded.zero_()
+        emo.zero_()
+
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            text = row[0]
+            text_padded[i, : text.size(0)] = text
+            text_lengths[i] = text.size(0)
+
+            spec = row[1]
+            spec_padded[i, :, : spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wav = row[2]
+            wav_padded[i, :, : wav.size(1)] = wav
+            wav_lengths[i] = wav.size(1)
+
+            sid[i] = row[3]
+
+            tone = row[4]
+            tone_padded[i, : tone.size(0)] = tone
+
+            language = row[5]
+            language_padded[i, : language.size(0)] = language
+
+            bert = row[6]
+            bert_padded[i, :, : bert.size(1)] = bert
+
+            ja_bert = row[7]
+            ja_bert_padded[i, :, : ja_bert.size(1)] = ja_bert
+
+            en_bert = row[8]
+            en_bert_padded[i, :, : en_bert.size(1)] = en_bert
+
+            emo[i, :] = row[9]
+
+        return (
+            text_padded,
+            text_lengths,
+            spec_padded,
+            spec_lengths,
+            wav_padded,
+            wav_lengths,
+            sid,
+            tone_padded,
+            language_padded,
+            bert_padded,
+            ja_bert_padded,
+            en_bert_padded,
+            emo,
+        )
+
+
+class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
+    """
+    Maintain similar input lengths in a batch.
+    Length groups are specified by boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
+
+    It removes samples which are not included in the boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
+    """
+
+    def __init__(
+        self,
+        dataset,
+        batch_size,
+        boundaries,
+        num_replicas=None,
+        rank=None,
+        shuffle=True,
+    ):
+        super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
+        self.lengths = dataset.lengths
+        self.batch_size = batch_size
+        self.boundaries = boundaries
+
+        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        self.total_size = sum(self.num_samples_per_bucket)
+        self.num_samples = self.total_size // self.num_replicas
+
+    def _create_buckets(self):
+        buckets = [[] for _ in range(len(self.boundaries) - 1)]
+        for i in range(len(self.lengths)):
+            length = self.lengths[i]
+            idx_bucket = self._bisect(length)
+            if idx_bucket != -1:
+                buckets[idx_bucket].append(i)
+
+        try:
+            for i in range(len(buckets) - 1, 0, -1):
+                if len(buckets[i]) == 0:
+                    buckets.pop(i)
+                    self.boundaries.pop(i + 1)
+            assert all(len(bucket) > 0 for bucket in buckets)
+        # When one bucket is not traversed
+        except Exception as e:
+            print("Bucket warning ", e)
+            for i in range(len(buckets) - 1, -1, -1):
+                if len(buckets[i]) == 0:
+                    buckets.pop(i)
+                    self.boundaries.pop(i + 1)
+
+        num_samples_per_bucket = []
+        for i in range(len(buckets)):
+            len_bucket = len(buckets[i])
+            total_batch_size = self.num_replicas * self.batch_size
+            rem = (
+                total_batch_size - (len_bucket % total_batch_size)
+            ) % total_batch_size
+            num_samples_per_bucket.append(len_bucket + rem)
+        return buckets, num_samples_per_bucket
+
+    def __iter__(self):
+        # deterministically shuffle based on epoch
+        g = torch.Generator()
+        g.manual_seed(self.epoch)
+
+        indices = []
+        if self.shuffle:
+            for bucket in self.buckets:
+                indices.append(torch.randperm(len(bucket), generator=g).tolist())
+        else:
+            for bucket in self.buckets:
+                indices.append(list(range(len(bucket))))
+
+        batches = []
+        for i in range(len(self.buckets)):
+            bucket = self.buckets[i]
+            len_bucket = len(bucket)
+            if len_bucket == 0:
+                continue
+            ids_bucket = indices[i]
+            num_samples_bucket = self.num_samples_per_bucket[i]
+
+            # add extra samples to make it evenly divisible
+            rem = num_samples_bucket - len_bucket
+            ids_bucket = (
+                ids_bucket
+                + ids_bucket * (rem // len_bucket)
+                + ids_bucket[: (rem % len_bucket)]
+            )
+
+            # subsample
+            ids_bucket = ids_bucket[self.rank :: self.num_replicas]
+
+            # batching
+            for j in range(len(ids_bucket) // self.batch_size):
+                batch = [
+                    bucket[idx]
+                    for idx in ids_bucket[
+                        j * self.batch_size : (j + 1) * self.batch_size
+                    ]
+                ]
+                batches.append(batch)
+
+        if self.shuffle:
+            batch_ids = torch.randperm(len(batches), generator=g).tolist()
+            batches = [batches[i] for i in batch_ids]
+        self.batches = batches
+
+        assert len(self.batches) * self.batch_size == self.num_samples
+        return iter(self.batches)
+
+    def _bisect(self, x, lo=0, hi=None):
+        if hi is None:
+            hi = len(self.boundaries) - 1
+
+        if hi > lo:
+            mid = (hi + lo) // 2
+            if self.boundaries[mid] < x and x <= self.boundaries[mid + 1]:
+                return mid
+            elif x <= self.boundaries[mid]:
+                return self._bisect(x, lo, mid)
+            else:
+                return self._bisect(x, mid + 1, hi)
+        else:
+            return -1
+
+    def __len__(self):
+        return self.num_samples // self.batch_size

default_config.yml

@@ -0,0 +1,176 @@
+# 全局配置
+# 对于希望在同一时间使用多个配置文件的情况，例如两个GPU同时跑两个训练集：通过环境变量指定配置文件，不指定则默认为./config.yml
+
+# 拟提供通用路径配置，统一存放数据，避免数据放得很乱
+# 每个数据集与其对应的模型存放至统一路径下，后续所有的路径配置均为相对于datasetPath的路径
+# 不填或者填空则路径为相对于项目根目录的路径
+dataset_path: "Data/"
+
+# 模型镜像源，默认huggingface，使用openi镜像源需指定openi_token
+mirror: ""
+openi_token: ""  # openi token
+
+# resample 音频重采样配置
+# 注意， “:” 后需要加空格
+resample:
+  # 目标重采样率
+  sampling_rate: 44100
+  # 音频文件输入路径，重采样会将该路径下所有.wav音频文件重采样
+  # 请填入相对于datasetPath的相对路径
+  in_dir: "audios/raw" # 相对于根目录的路径为 /datasetPath/in_dir
+  # 音频文件重采样后输出路径
+  out_dir: "audios/wavs"
+
+
+# preprocess_text 数据集预处理相关配置
+# 注意， “:” 后需要加空格
+preprocess_text:
+  # 原始文本文件路径，文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
+  transcription_path: "filelists/你的数据集文本.list"
+  # 数据清洗后文本路径，可以不填。不填则将在原始文本目录生成
+  cleaned_path: ""
+  # 训练集路径
+  train_path: "filelists/train.list"
+  # 验证集路径
+  val_path: "filelists/val.list"
+  # 配置文件路径
+  config_path: "config.json"
+  # 每个speaker的验证集条数
+  val_per_spk: 4
+  # 验证集最大条数，多于的会被截断并放到训练集中
+  max_val_total: 8
+  # 是否进行数据清洗
+  clean: true
+
+
+# bert_gen 相关配置
+# 注意， “:” 后需要加空格
+bert_gen:
+  # 训练数据集配置文件路径
+  config_path: "config.json"
+  # 并行数
+  num_processes: 2
+  # 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
+  # 该选项同时决定了get_bert_feature的默认设备
+  device: "cuda"
+  # 使用多卡推理
+  use_multi_device: false
+
+# emo_gen 相关配置
+# 注意， “:” 后需要加空格
+emo_gen:
+  # 训练数据集配置文件路径
+  config_path: "config.json"
+  # 并行数
+  num_processes: 2
+  # 使用设备：可选项 "cuda" 显卡推理，"cpu" cpu推理
+  device: "cuda"
+
+# train 训练配置
+# 注意， “:” 后需要加空格
+train_ms:
+  env:
+    MASTER_ADDR: "localhost"
+    MASTER_PORT: 10086
+    WORLD_SIZE: 1
+    LOCAL_RANK: 0
+    RANK: 0
+    # 可以填写任意名的环境变量
+    # THE_ENV_VAR_YOU_NEED_TO_USE: "1234567"
+  # 底模设置
+  base:
+    use_base_model: false
+    repo_id: "Stardust_minus/Bert-VITS2"
+    model_image: "Bert-VITS2_2.1-Emo底模" # openi网页的模型名
+  # 训练模型存储目录：与旧版本的区别，原先数据集是存放在logs/model_name下的，现在改为统一存放在Data/你的数据集/models下
+  model: "models"
+  # 配置文件路径
+  config_path: "config.json"
+  # 训练使用的worker，不建议超过CPU核心数
+  num_workers: 16
+  # 关闭此项可以节约接近50%的磁盘空间，但是可能导致实际训练速度变慢和更高的CPU使用率。
+  spec_cache: True
+  # 保存的检查点数量，多于此数目的权重会被删除来节省空间。
+  keep_ckpts: 8
+
+
+# webui webui配置
+# 注意， “:” 后需要加空格
+webui:
+  # 推理设备
+  device: "cuda"
+  # 模型路径
+  model: "genshin/models/G_8000.pth"
+  # 配置文件路径
+  config_path: "config.json"
+  # 端口号
+  port: 7860
+  # 是否公开部署，对外网开放
+  share: false
+  # 是否开启debug模式
+  debug: false
+  # 语种识别库，可选langid, fastlid
+  language_identification_library: "langid"
+
+
+# server api配置
+# 注意， “:” 后需要加空格
+# 注意，本配置下的所有配置均为相对于根目录的路径
+server:
+  # 端口号
+  port: 5000
+  # 模型默认使用设备：但是当前并没有实现这个配置。
+  device: "cuda"
+  # 需要加载的所有模型的配置，可以填多个模型，也可以不填模型，等网页成功后手动加载模型
+  # 不加载模型的配置格式：删除默认给的两个模型配置，给models赋值 [ ]，也就是空列表。参考模型2的speakers 即 models: [ ]
+  # 注意，所有模型都必须正确配置model与config的路径，空路径会导致加载错误。
+  # 也可以不填模型，等网页加载成功后手动填写models。
+  models:
+    - # 模型的路径
+      model: ""
+      # 模型config.json的路径
+      config: ""
+      # 模型使用设备，若填写则会覆盖默认配置
+      device: "cuda"
+      # 模型默认使用的语言
+      language: "ZH"
+      # 模型人物默认参数
+      # 不必填写所有人物，不填的使用默认值
+      # 暂时不用填写，当前尚未实现按人区分配置
+      speakers:
+        - speaker: "科比"
+          sdp_ratio: 0.2
+          noise_scale: 0.6
+          noise_scale_w: 0.8
+          length_scale: 1
+        - speaker: "五条悟"
+          sdp_ratio: 0.3
+          noise_scale: 0.7
+          noise_scale_w: 0.8
+          length_scale: 0.5
+        - speaker: "安倍晋三"
+          sdp_ratio: 0.2
+          noise_scale: 0.6
+          noise_scale_w: 0.8
+          length_scale: 1.2
+    - # 模型的路径
+      model: ""
+      # 模型config.json的路径
+      config: ""
+      # 模型使用设备，若填写则会覆盖默认配置
+      device: "cpu"
+      # 模型默认使用的语言
+      language: "JP"
+      # 模型人物默认参数
+      # 不必填写所有人物，不填的使用默认值
+      speakers: [ ] # 也可以不填
+
+
+# 百度翻译开放平台 api配置
+# api接入文档 https://api.fanyi.baidu.com/doc/21
+# 请不要在github等网站公开分享你的app id 与 key
+translate:
+  # 你的APPID
+  "app_key": ""
+  # 你的密钥
+  "secret_key": ""

emo_gen.py

@@ -0,0 +1,155 @@
+import argparse
+import os
+from pathlib import Path
+
+import librosa
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+from transformers import Wav2Vec2Processor
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2Model,
+    Wav2Vec2PreTrainedModel,
+)
+
+import utils
+from config import config
+
+
+class RegressionHead(nn.Module):
+    r"""Classification head."""
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.final_dropout)
+        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+
+        return x
+
+
+class EmotionModel(Wav2Vec2PreTrainedModel):
+    r"""Speech emotion classifier."""
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.classifier = RegressionHead(config)
+        self.init_weights()
+
+    def forward(
+        self,
+        input_values,
+    ):
+        outputs = self.wav2vec2(input_values)
+        hidden_states = outputs[0]
+        hidden_states = torch.mean(hidden_states, dim=1)
+        logits = self.classifier(hidden_states)
+
+        return hidden_states, logits
+
+
+class AudioDataset(Dataset):
+    def __init__(self, list_of_wav_files, sr, processor):
+        self.list_of_wav_files = list_of_wav_files
+        self.processor = processor
+        self.sr = sr
+
+    def __len__(self):
+        return len(self.list_of_wav_files)
+
+    def __getitem__(self, idx):
+        wav_file = self.list_of_wav_files[idx]
+        audio_data, _ = librosa.load(wav_file, sr=self.sr)
+        processed_data = self.processor(audio_data, sampling_rate=self.sr)[
+            "input_values"
+        ][0]
+        return torch.from_numpy(processed_data)
+
+
+def process_func(
+    x: np.ndarray,
+    sampling_rate: int,
+    model: EmotionModel,
+    processor: Wav2Vec2Processor,
+    device: str,
+    embeddings: bool = False,
+) -> np.ndarray:
+    r"""Predict emotions or extract embeddings from raw audio signal."""
+    model = model.to(device)
+    y = processor(x, sampling_rate=sampling_rate)
+    y = y["input_values"][0]
+    y = torch.from_numpy(y).unsqueeze(0).to(device)
+
+    # run through model
+    with torch.no_grad():
+        y = model(y)[0 if embeddings else 1]
+
+    # convert to numpy
+    y = y.detach().cpu().numpy()
+
+    return y
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c", "--config", type=str, default=config.bert_gen_config.config_path
+    )
+    parser.add_argument(
+        "--num_processes", type=int, default=config.bert_gen_config.num_processes
+    )
+    args, _ = parser.parse_known_args()
+    config_path = args.config
+    hps = utils.get_hparams_from_file(config_path)
+
+    device = config.bert_gen_config.device
+
+    model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
+    REPO_ID = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim"
+    if not Path(model_name).joinpath("pytorch_model.bin").exists():
+        utils.download_emo_models(config.mirror, REPO_ID, model_name)
+
+    processor = Wav2Vec2Processor.from_pretrained(model_name)
+    model = EmotionModel.from_pretrained(model_name).to(device)
+
+    lines = []
+    with open(hps.data.training_files, encoding="utf-8") as f:
+        lines.extend(f.readlines())
+
+    with open(hps.data.validation_files, encoding="utf-8") as f:
+        lines.extend(f.readlines())
+
+    wavnames = [line.split("|")[0] for line in lines]
+    dataset = AudioDataset(wavnames, 16000, processor)
+    data_loader = DataLoader(
+        dataset,
+        batch_size=1,
+        shuffle=False,
+        num_workers=min(args.num_processes, os.cpu_count() - 1),
+    )
+
+    with torch.no_grad():
+        for i, data in tqdm(enumerate(data_loader), total=len(data_loader)):
+            wavname = wavnames[i]
+            emo_path = wavname.replace(".wav", ".emo.npy")
+            if os.path.exists(emo_path):
+                continue
+            emb = model(data.to(device))[0].detach().cpu().numpy()
+            np.save(emo_path, emb)
+
+    print("Emo vec 生成完毕!")

export_onnx.py

@@ -0,0 +1,56 @@
+from models_onnx import SynthesizerTrn
+import utils
+from text.symbols import symbols
+import os
+import json
+
+
+def export_onnx(export_path, model_path, config_path):
+    hps = utils.get_hparams_from_file(config_path)
+    net_g = SynthesizerTrn(
+        len(symbols),
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=hps.data.n_speakers,
+        **hps.model,
+    )
+    _ = net_g.eval()
+    _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
+    net_g.export_onnx(export_path)
+
+    spklist = []
+    for key in hps.data.spk2id.keys():
+        spklist.append(key)
+
+    MoeVSConf = {
+        "Folder": f"{export_path}",
+        "Name": f"{export_path}",
+        "Type": "BertVits",
+        "Symbol": symbols,
+        "Cleaner": "",
+        "Rate": hps.data.sampling_rate,
+        "CharaMix": True,
+        "Characters": spklist,
+        "LanguageMap": {"ZH": [0, 0], "JP": [1, 6], "EN": [2, 8]},
+        "Dict": "BasicDict",
+        "BertPath": [
+            "chinese-roberta-wwm-ext-large",
+            "deberta-v2-large-japanese",
+            "bert-base-japanese-v3",
+        ],
+    }
+
+    with open(f"onnx/{export_path}.json", "w") as MoeVsConfFile:
+        json.dump(MoeVSConf, MoeVsConfFile, indent=4)
+
+
+if __name__ == "__main__":
+    print(symbols)
+    export_path = "HimenoSena"
+    model_path = "G_53000.pth"
+    config_path = "config.json"
+    if not os.path.exists("onnx"):
+        os.makedirs("onnx")
+    if not os.path.exists(f"onnx/{export_path}"):
+        os.makedirs(f"onnx/{export_path}")
+    export_onnx(export_path, model_path, config_path)

get_emo.py

@@ -0,0 +1,26 @@
+from emo_gen import EmotionModel, process_func
+
+import librosa
+import numpy as np
+import torch
+from transformers import Wav2Vec2Processor
+
+from config import config
+
+model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
+device = "cuda" if torch.cuda.is_available() else "cpu"
+processor = Wav2Vec2Processor.from_pretrained(model_name)
+model = EmotionModel.from_pretrained(model_name).to(device)
+
+
+def get_emo(path):
+    wav, sr = librosa.load(path, 16000)
+    device = config.bert_gen_config.device
+    return process_func(
+        np.expand_dims(wav, 0).astype(np.float64),
+        sr,
+        model,
+        processor,
+        device,
+        embeddings=True,
+    ).squeeze(0)

infer.py

@@ -0,0 +1,341 @@
+"""
+版本管理、兼容推理及模型加载实现。
+版本说明：
+    1. 版本号与github的release版本号对应，使用哪个release版本训练的模型即对应其版本号
+    2. 请在模型的config.json中显示声明版本号，添加一个字段"version" : "你的版本号"
+特殊版本说明：
+    1.1.1-fix： 1.1.1版本训练的模型，但是在推理时使用dev的日语修复
+    1.1.1-dev： dev开发
+    2.1：当前版本
+"""
+import torch
+import commons
+from text import cleaned_text_to_sequence, get_bert
+from get_emo import get_emo
+from text.cleaner import clean_text
+import utils
+
+from models import SynthesizerTrn
+from text.symbols import symbols
+from oldVersion.V200.models import SynthesizerTrn as V200SynthesizerTrn
+from oldVersion.V200.text import symbols as V200symbols
+from oldVersion.V111.models import SynthesizerTrn as V111SynthesizerTrn
+from oldVersion.V111.text import symbols as V111symbols
+from oldVersion.V110.models import SynthesizerTrn as V110SynthesizerTrn
+from oldVersion.V110.text import symbols as V110symbols
+from oldVersion.V101.models import SynthesizerTrn as V101SynthesizerTrn
+from oldVersion.V101.text import symbols as V101symbols
+
+from oldVersion import V111, V110, V101, V200
+
+# 当前版本信息
+latest_version = "2.1"
+
+# 版本兼容
+SynthesizerTrnMap = {
+    "2.0.2-fix": V200SynthesizerTrn,
+    "2.0.1": V200SynthesizerTrn,
+    "2.0": V200SynthesizerTrn,
+    "1.1.1-fix": V111SynthesizerTrn,
+    "1.1.1": V111SynthesizerTrn,
+    "1.1": V110SynthesizerTrn,
+    "1.1.0": V110SynthesizerTrn,
+    "1.0.1": V101SynthesizerTrn,
+    "1.0": V101SynthesizerTrn,
+    "1.0.0": V101SynthesizerTrn,
+}
+
+symbolsMap = {
+    "2.0.2-fix": V200symbols,
+    "2.0.1": V200symbols,
+    "2.0": V200symbols,
+    "1.1.1-fix": V111symbols,
+    "1.1.1": V111symbols,
+    "1.1": V110symbols,
+    "1.1.0": V110symbols,
+    "1.0.1": V101symbols,
+    "1.0": V101symbols,
+    "1.0.0": V101symbols,
+}
+
+
+def get_net_g(model_path: str, version: str, device: str, hps):
+    if version != latest_version:
+        net_g = SynthesizerTrnMap[version](
+            len(symbolsMap[version]),
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            n_speakers=hps.data.n_speakers,
+            **hps.model,
+        ).to(device)
+    else:
+        # 当前版本模型 net_g
+        net_g = SynthesizerTrn(
+            len(symbols),
+            hps.data.filter_length // 2 + 1,
+            hps.train.segment_size // hps.data.hop_length,
+            n_speakers=hps.data.n_speakers,
+            **hps.model,
+        ).to(device)
+    _ = net_g.eval()
+    _ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
+    return net_g
+
+
+def get_text(text, language_str, hps, device):
+    # 在此处实现当前版本的get_text
+    norm_text, phone, tone, word2ph = clean_text(text, language_str)
+    phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
+
+    if hps.data.add_blank:
+        phone = commons.intersperse(phone, 0)
+        tone = commons.intersperse(tone, 0)
+        language = commons.intersperse(language, 0)
+        for i in range(len(word2ph)):
+            word2ph[i] = word2ph[i] * 2
+        word2ph[0] += 1
+    bert_ori = get_bert(norm_text, word2ph, language_str, device)
+    del word2ph
+    assert bert_ori.shape[-1] == len(phone), phone
+
+    if language_str == "ZH":
+        bert = bert_ori
+        ja_bert = torch.zeros(1024, len(phone))
+        en_bert = torch.zeros(1024, len(phone))
+    elif language_str == "JP":
+        bert = torch.zeros(1024, len(phone))
+        ja_bert = bert_ori
+        en_bert = torch.zeros(1024, len(phone))
+    elif language_str == "EN":
+        bert = torch.zeros(1024, len(phone))
+        ja_bert = torch.zeros(1024, len(phone))
+        en_bert = bert_ori
+    else:
+        raise ValueError("language_str should be ZH, JP or EN")
+
+    assert bert.shape[-1] == len(
+        phone
+    ), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
+
+    phone = torch.LongTensor(phone)
+    tone = torch.LongTensor(tone)
+    language = torch.LongTensor(language)
+    return bert, ja_bert, en_bert, phone, tone, language
+
+
+def get_emo_(reference_audio, emotion):
+    emo = (
+        torch.from_numpy(get_emo(reference_audio))
+        if reference_audio
+        else torch.Tensor([emotion])
+    )
+    return emo
+
+
+def infer(
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    language,
+    hps,
+    net_g,
+    device,
+    reference_audio=None,
+    emotion=None,
+    skip_start=False,
+    skip_end=False,
+):
+    # 支持中日英三语版本
+    inferMap_V2 = {
+        "2.0.2-fix": V200.infer,
+        "2.0.1": V200.infer,
+        "2.0": V200.infer,
+        "1.1.1-fix": V111.infer_fix,
+        "1.1.1": V111.infer,
+        "1.1": V110.infer,
+        "1.1.0": V110.infer,
+    }
+    # 仅支持中文版本
+    # 在测试中，并未发现两��版本的模型不能互相通用
+    inferMap_V1 = {
+        "1.0.1": V101.infer,
+        "1.0": V101.infer,
+        "1.0.0": V101.infer,
+    }
+    version = hps.version if hasattr(hps, "version") else latest_version
+    # 非当前版本，根据版本号选择合适的infer
+    if version != latest_version:
+        if version in inferMap_V2.keys():
+            return inferMap_V2[version](
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                language,
+                hps,
+                net_g,
+                device,
+            )
+        if version in inferMap_V1.keys():
+            return inferMap_V1[version](
+                text,
+                sdp_ratio,
+                noise_scale,
+                noise_scale_w,
+                length_scale,
+                sid,
+                hps,
+                net_g,
+                device,
+            )
+    # 在此处实现当前版本的推理
+    bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
+        text, language, hps, device
+    )
+    emo = get_emo_(reference_audio, emotion)
+    if skip_start:
+        phones = phones[1:]
+        tones = tones[1:]
+        lang_ids = lang_ids[1:]
+        bert = bert[:, 1:]
+        ja_bert = ja_bert[:, 1:]
+        en_bert = en_bert[:, 1:]
+    if skip_end:
+        phones = phones[:-1]
+        tones = tones[:-1]
+        lang_ids = lang_ids[:-1]
+        bert = bert[:, :-1]
+        ja_bert = ja_bert[:, :-1]
+        en_bert = en_bert[:, :-1]
+    with torch.no_grad():
+        x_tst = phones.to(device).unsqueeze(0)
+        tones = tones.to(device).unsqueeze(0)
+        lang_ids = lang_ids.to(device).unsqueeze(0)
+        bert = bert.to(device).unsqueeze(0)
+        ja_bert = ja_bert.to(device).unsqueeze(0)
+        en_bert = en_bert.to(device).unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+        emo = emo.to(device).unsqueeze(0)
+        del phones
+        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        audio = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                speakers,
+                tones,
+                lang_ids,
+                bert,
+                ja_bert,
+                en_bert,
+                emo,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+            )[0][0, 0]
+            .data.cpu()
+            .float()
+            .numpy()
+        )
+        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio
+
+
+def infer_multilang(
+    text,
+    sdp_ratio,
+    noise_scale,
+    noise_scale_w,
+    length_scale,
+    sid,
+    language,
+    hps,
+    net_g,
+    device,
+    reference_audio=None,
+    emotion=None,
+    skip_start=False,
+    skip_end=False,
+):
+    bert, ja_bert, en_bert, phones, tones, lang_ids = [], [], [], [], [], []
+    emo = get_emo_(reference_audio, emotion)
+    for idx, (txt, lang) in enumerate(zip(text, language)):
+        skip_start = (idx != 0) or (skip_start and idx == 0)
+        skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
+        (
+            temp_bert,
+            temp_ja_bert,
+            temp_en_bert,
+            temp_phones,
+            temp_tones,
+            temp_lang_ids,
+        ) = get_text(txt, lang, hps, device)
+        if skip_start:
+            temp_bert = temp_bert[:, 1:]
+            temp_ja_bert = temp_ja_bert[:, 1:]
+            temp_en_bert = temp_en_bert[:, 1:]
+            temp_phones = temp_phones[1:]
+            temp_tones = temp_tones[1:]
+            temp_lang_ids = temp_lang_ids[1:]
+        if skip_end:
+            temp_bert = temp_bert[:, :-1]
+            temp_ja_bert = temp_ja_bert[:, :-1]
+            temp_en_bert = temp_en_bert[:, :-1]
+            temp_phones = temp_phones[:-1]
+            temp_tones = temp_tones[:-1]
+            temp_lang_ids = temp_lang_ids[:-1]
+        bert.append(temp_bert)
+        ja_bert.append(temp_ja_bert)
+        en_bert.append(temp_en_bert)
+        phones.append(temp_phones)
+        tones.append(temp_tones)
+        lang_ids.append(temp_lang_ids)
+    bert = torch.concatenate(bert, dim=1)
+    ja_bert = torch.concatenate(ja_bert, dim=1)
+    en_bert = torch.concatenate(en_bert, dim=1)
+    phones = torch.concatenate(phones, dim=0)
+    tones = torch.concatenate(tones, dim=0)
+    lang_ids = torch.concatenate(lang_ids, dim=0)
+    with torch.no_grad():
+        x_tst = phones.to(device).unsqueeze(0)
+        tones = tones.to(device).unsqueeze(0)
+        lang_ids = lang_ids.to(device).unsqueeze(0)
+        bert = bert.to(device).unsqueeze(0)
+        ja_bert = ja_bert.to(device).unsqueeze(0)
+        en_bert = en_bert.to(device).unsqueeze(0)
+        emo = emo.to(device).unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
+        del phones
+        speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
+        audio = (
+            net_g.infer(
+                x_tst,
+                x_tst_lengths,
+                speakers,
+                tones,
+                lang_ids,
+                bert,
+                ja_bert,
+                en_bert,
+                emo,
+                sdp_ratio=sdp_ratio,
+                noise_scale=noise_scale,
+                noise_scale_w=noise_scale_w,
+                length_scale=length_scale,
+            )[0][0, 0]
+            .data.cpu()
+            .float()
+            .numpy()
+        )
+        del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio