init

G_49200.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da931601660595a5ab841b25668faf97ee66e10d3ae5da03f69c5e61f28476fd
+size 479164585

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Jaehyeon Kim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

all_emotions.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48e81667f1fc4ce2b2eaed80fadd0871e1ddfc8933767915954c39ac854d5724
+size 22356096

app.py

@@ -0,0 +1,111 @@
+import gradio as gr
+import torch
+import commons
+import utils
+from models import SynthesizerTrn
+from text.symbols import symbols
+from text import text_to_sequence
+import numpy as np
+
+
+def get_text(text, hps):
+    text_norm = text_to_sequence(text, hps.data.text_cleaners)
+    if hps.data.add_blank:
+        text_norm = commons.intersperse(text_norm, 0)
+    text_norm = torch.LongTensor(text_norm)
+    return text_norm
+hps = utils.get_hparams_from_file("./configs/vtubers.json")
+net_g = SynthesizerTrn(
+    len(symbols),
+    hps.data.filter_length // 2 + 1,
+    hps.train.segment_size // hps.data.hop_length,
+    n_speakers=hps.data.n_speakers,
+    **hps.model)
+_ = net_g.eval()
+
+_ = utils.load_checkpoint("./G_49200.pth", net_g, None)
+all_emotions = np.load("all_emotions.npy")
+emotion_dict = {
+    "小声": 2077,
+    "激动": 111,
+    "平静1": 434,
+    "平静2": 3554
+}
+import random
+def tts(txt, emotion):
+    stn_tst = get_text(txt, hps)
+    randsample = None
+    with torch.no_grad():
+        x_tst = stn_tst.unsqueeze(0)
+        x_tst_lengths = torch.LongTensor([stn_tst.size(0)])
+        sid = torch.LongTensor([0])
+        if type(emotion) ==int:
+            emo = torch.FloatTensor(all_emotions[emotion]).unsqueeze(0)
+        elif emotion == "random":
+            emo = torch.randn([1,1024])
+        elif emotion == "random_sample":
+            randint = random.randint(0, all_emotions.shape[0])
+            emo = torch.FloatTensor(all_emotions[randint]).unsqueeze(0)
+            randsample = randint
+        elif emotion.endswith("wav"):
+            import emotion_extract
+            emo = torch.FloatTensor(emotion_extract.extract_wav(emotion))
+        else:
+            emo = torch.FloatTensor(all_emotions[emotion_dict[emotion]]).unsqueeze(0)
+
+        audio = net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=0.667, noise_scale_w=0.8, length_scale=1, emo=emo)[0][0,0].data.float().numpy()
+    return audio, randsample
+
+
+def tts1(text, emotion):
+    if len(text) > 150:
+        return "Error: Text is too long", None
+    audio, _ = tts(text, emotion)
+    return "Success", (hps.data.sampling_rate, audio)
+
+def tts2(text):
+    if len(text) > 150:
+        return "Error: Text is too long", None
+    audio, randsample = tts(text, "random_sample")
+
+    return str(randsample), (hps.data.sampling_rate, audio)
+
+def tts3(text, sample):
+    if len(text) > 150:
+        return "Error: Text is too long", None
+    try:
+        audio, _ = tts(text, int(sample))
+        return "Success", (hps.data.sampling_rate, audio)
+    except:
+        return "输入参数不为整数或其他错误"
+
+app = gr.Blocks()
+with app:
+    with gr.Tabs():
+        with gr.TabItem("使用预制情感合成"):
+            tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
+            tts_input2 = gr.Dropdown(label="情感", choices=list(emotion_dict.keys()),  value="平静1")
+            tts_submit = gr.Button("合成音频", variant="primary")
+            tts_output1 = gr.Textbox(label="Message")
+            tts_output2 = gr.Audio(label="Output")
+            tts_submit.click(tts1, [tts_input1, tts_input2], [tts_output1, tts_output2])
+        with gr.TabItem("随机抽取训练集样本作为情感参数"):
+            tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
+            tts_submit = gr.Button("合成音频", variant="primary")
+            tts_output1 = gr.Textbox(label="随机样本id（可用于第三个tab中合成）")
+            tts_output2 = gr.Audio(label="Output")
+            tts_submit.click(tts2, [tts_input1], [tts_output1, tts_output2])
+
+        with gr.TabItem("使用情感样本id作为情感参数"):
+
+            tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
+            tts_input2 = gr.Number(label="情感样本id", value=2004)
+            tts_submit = gr.Button("合成音频", variant="primary")
+            tts_output1 = gr.Textbox(label="Message")
+            tts_output2 = gr.Audio(label="Output")
+            tts_submit.click(tts3, [tts_input1, tts_input2], [tts_output1, tts_output2])
+
+        with gr.TabItem("使用参考音频作为情感参数"):
+            tts_input1 = gr.TextArea(label="text", value="暂未实现")
+
+    app.launch()

attentions.py

@@ -0,0 +1,303 @@
+import copy
+import math
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import commons
+import modules
+from modules import LayerNorm
+   
+
+class Encoder(nn.Module):
+  def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
+    super().__init__()
+    self.hidden_channels = hidden_channels
+    self.filter_channels = filter_channels
+    self.n_heads = n_heads
+    self.n_layers = n_layers
+    self.kernel_size = kernel_size
+    self.p_dropout = p_dropout
+    self.window_size = window_size
+
+    self.drop = nn.Dropout(p_dropout)
+    self.attn_layers = nn.ModuleList()
+    self.norm_layers_1 = nn.ModuleList()
+    self.ffn_layers = nn.ModuleList()
+    self.norm_layers_2 = nn.ModuleList()
+    for i in range(self.n_layers):
+      self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
+      self.norm_layers_1.append(LayerNorm(hidden_channels))
+      self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
+      self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+  def forward(self, x, x_mask):
+    attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+    x = x * x_mask
+    for i in range(self.n_layers):
+      y = self.attn_layers[i](x, x, attn_mask)
+      y = self.drop(y)
+      x = self.norm_layers_1[i](x + y)
+
+      y = self.ffn_layers[i](x, x_mask)
+      y = self.drop(y)
+      x = self.norm_layers_2[i](x + y)
+    x = x * x_mask
+    return x
+
+
+class Decoder(nn.Module):
+  def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
+    super().__init__()
+    self.hidden_channels = hidden_channels
+    self.filter_channels = filter_channels
+    self.n_heads = n_heads
+    self.n_layers = n_layers
+    self.kernel_size = kernel_size
+    self.p_dropout = p_dropout
+    self.proximal_bias = proximal_bias
+    self.proximal_init = proximal_init
+
+    self.drop = nn.Dropout(p_dropout)
+    self.self_attn_layers = nn.ModuleList()
+    self.norm_layers_0 = nn.ModuleList()
+    self.encdec_attn_layers = nn.ModuleList()
+    self.norm_layers_1 = nn.ModuleList()
+    self.ffn_layers = nn.ModuleList()
+    self.norm_layers_2 = nn.ModuleList()
+    for i in range(self.n_layers):
+      self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
+      self.norm_layers_0.append(LayerNorm(hidden_channels))
+      self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
+      self.norm_layers_1.append(LayerNorm(hidden_channels))
+      self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
+      self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+  def forward(self, x, x_mask, h, h_mask):
+    """
+    x: decoder input
+    h: encoder output
+    """
+    self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
+    encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+    x = x * x_mask
+    for i in range(self.n_layers):
+      y = self.self_attn_layers[i](x, x, self_attn_mask)
+      y = self.drop(y)
+      x = self.norm_layers_0[i](x + y)
+
+      y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+      y = self.drop(y)
+      x = self.norm_layers_1[i](x + y)
+      
+      y = self.ffn_layers[i](x, x_mask)
+      y = self.drop(y)
+      x = self.norm_layers_2[i](x + y)
+    x = x * x_mask
+    return x
+
+
+class MultiHeadAttention(nn.Module):
+  def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
+    super().__init__()
+    assert channels % n_heads == 0
+
+    self.channels = channels
+    self.out_channels = out_channels
+    self.n_heads = n_heads
+    self.p_dropout = p_dropout
+    self.window_size = window_size
+    self.heads_share = heads_share
+    self.block_length = block_length
+    self.proximal_bias = proximal_bias
+    self.proximal_init = proximal_init
+    self.attn = None
+
+    self.k_channels = channels // n_heads
+    self.conv_q = nn.Conv1d(channels, channels, 1)
+    self.conv_k = nn.Conv1d(channels, channels, 1)
+    self.conv_v = nn.Conv1d(channels, channels, 1)
+    self.conv_o = nn.Conv1d(channels, out_channels, 1)
+    self.drop = nn.Dropout(p_dropout)
+
+    if window_size is not None:
+      n_heads_rel = 1 if heads_share else n_heads
+      rel_stddev = self.k_channels**-0.5
+      self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
+      self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
+
+    nn.init.xavier_uniform_(self.conv_q.weight)
+    nn.init.xavier_uniform_(self.conv_k.weight)
+    nn.init.xavier_uniform_(self.conv_v.weight)
+    if proximal_init:
+      with torch.no_grad():
+        self.conv_k.weight.copy_(self.conv_q.weight)
+        self.conv_k.bias.copy_(self.conv_q.bias)
+      
+  def forward(self, x, c, attn_mask=None):
+    q = self.conv_q(x)
+    k = self.conv_k(c)
+    v = self.conv_v(c)
+    
+    x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+    x = self.conv_o(x)
+    return x
+
+  def attention(self, query, key, value, mask=None):
+    # reshape [b, d, t] -> [b, n_h, t, d_k]
+    b, d, t_s, t_t = (*key.size(), query.size(2))
+    query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+    key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+    value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+    scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+    if self.window_size is not None:
+      assert t_s == t_t, "Relative attention is only available for self-attention."
+      key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
+      rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
+      scores_local = self._relative_position_to_absolute_position(rel_logits)
+      scores = scores + scores_local
+    if self.proximal_bias:
+      assert t_s == t_t, "Proximal bias is only available for self-attention."
+      scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
+    if mask is not None:
+      scores = scores.masked_fill(mask == 0, -1e4)
+      if self.block_length is not None:
+        assert t_s == t_t, "Local attention is only available for self-attention."
+        block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
+        scores = scores.masked_fill(block_mask == 0, -1e4)
+    p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
+    p_attn = self.drop(p_attn)
+    output = torch.matmul(p_attn, value)
+    if self.window_size is not None:
+      relative_weights = self._absolute_position_to_relative_position(p_attn)
+      value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
+      output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
+    output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
+    return output, p_attn
+
+  def _matmul_with_relative_values(self, x, y):
+    """
+    x: [b, h, l, m]
+    y: [h or 1, m, d]
+    ret: [b, h, l, d]
+    """
+    ret = torch.matmul(x, y.unsqueeze(0))
+    return ret
+
+  def _matmul_with_relative_keys(self, x, y):
+    """
+    x: [b, h, l, d]
+    y: [h or 1, m, d]
+    ret: [b, h, l, m]
+    """
+    ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+    return ret
+
+  def _get_relative_embeddings(self, relative_embeddings, length):
+    max_relative_position = 2 * self.window_size + 1
+    # Pad first before slice to avoid using cond ops.
+    pad_length = max(length - (self.window_size + 1), 0)
+    slice_start_position = max((self.window_size + 1) - length, 0)
+    slice_end_position = slice_start_position + 2 * length - 1
+    if pad_length > 0:
+      padded_relative_embeddings = F.pad(
+          relative_embeddings,
+          commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
+    else:
+      padded_relative_embeddings = relative_embeddings
+    used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
+    return used_relative_embeddings
+
+  def _relative_position_to_absolute_position(self, x):
+    """
+    x: [b, h, l, 2*l-1]
+    ret: [b, h, l, l]
+    """
+    batch, heads, length, _ = x.size()
+    # Concat columns of pad to shift from relative to absolute indexing.
+    x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
+
+    # Concat extra elements so to add up to shape (len+1, 2*len-1).
+    x_flat = x.view([batch, heads, length * 2 * length])
+    x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
+
+    # Reshape and slice out the padded elements.
+    x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
+    return x_final
+
+  def _absolute_position_to_relative_position(self, x):
+    """
+    x: [b, h, l, l]
+    ret: [b, h, l, 2*l-1]
+    """
+    batch, heads, length, _ = x.size()
+    # padd along column
+    x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
+    x_flat = x.view([batch, heads, length**2 + length*(length -1)])
+    # add 0's in the beginning that will skew the elements after reshape
+    x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+    x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
+    return x_final
+
+  def _attention_bias_proximal(self, length):
+    """Bias for self-attention to encourage attention to close positions.
+    Args:
+      length: an integer scalar.
+    Returns:
+      a Tensor with shape [1, 1, length, length]
+    """
+    r = torch.arange(length, dtype=torch.float32)
+    diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+    return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+  def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
+    super().__init__()
+    self.in_channels = in_channels
+    self.out_channels = out_channels
+    self.filter_channels = filter_channels
+    self.kernel_size = kernel_size
+    self.p_dropout = p_dropout
+    self.activation = activation
+    self.causal = causal
+
+    if causal:
+      self.padding = self._causal_padding
+    else:
+      self.padding = self._same_padding
+
+    self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+    self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+    self.drop = nn.Dropout(p_dropout)
+
+  def forward(self, x, x_mask):
+    x = self.conv_1(self.padding(x * x_mask))
+    if self.activation == "gelu":
+      x = x * torch.sigmoid(1.702 * x)
+    else:
+      x = torch.relu(x)
+    x = self.drop(x)
+    x = self.conv_2(self.padding(x * x_mask))
+    return x * x_mask
+  
+  def _causal_padding(self, x):
+    if self.kernel_size == 1:
+      return x
+    pad_l = self.kernel_size - 1
+    pad_r = 0
+    padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+    x = F.pad(x, commons.convert_pad_shape(padding))
+    return x
+
+  def _same_padding(self, x):
+    if self.kernel_size == 1:
+      return x
+    pad_l = (self.kernel_size - 1) // 2
+    pad_r = self.kernel_size // 2
+    padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+    x = F.pad(x, commons.convert_pad_shape(padding))
+    return x

commons.py

@@ -0,0 +1,161 @@
+import math
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def init_weights(m, mean=0.0, std=0.01):
+  classname = m.__class__.__name__
+  if classname.find("Conv") != -1:
+    m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+  return int((kernel_size*dilation - dilation)/2)
+
+
+def convert_pad_shape(pad_shape):
+  l = pad_shape[::-1]
+  pad_shape = [item for sublist in l for item in sublist]
+  return pad_shape
+
+
+def intersperse(lst, item):
+  result = [item] * (len(lst) * 2 + 1)
+  result[1::2] = lst
+  return result
+
+
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+  """KL(P||Q)"""
+  kl = (logs_q - logs_p) - 0.5
+  kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
+  return kl
+
+
+def rand_gumbel(shape):
+  """Sample from the Gumbel distribution, protect from overflows."""
+  uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
+  return -torch.log(-torch.log(uniform_samples))
+
+
+def rand_gumbel_like(x):
+  g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
+  return g
+
+
+def slice_segments(x, ids_str, segment_size=4):
+  ret = torch.zeros_like(x[:, :, :segment_size])
+  for i in range(x.size(0)):
+    idx_str = ids_str[i]
+    idx_end = idx_str + segment_size
+    ret[i] = x[i, :, idx_str:idx_end]
+  return ret
+
+
+def rand_slice_segments(x, x_lengths=None, segment_size=4):
+  b, d, t = x.size()
+  if x_lengths is None:
+    x_lengths = t
+  ids_str_max = x_lengths - segment_size + 1
+  ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
+  ret = slice_segments(x, ids_str, segment_size)
+  return ret, ids_str
+
+
+def get_timing_signal_1d(
+    length, channels, min_timescale=1.0, max_timescale=1.0e4):
+  position = torch.arange(length, dtype=torch.float)
+  num_timescales = channels // 2
+  log_timescale_increment = (
+      math.log(float(max_timescale) / float(min_timescale)) /
+      (num_timescales - 1))
+  inv_timescales = min_timescale * torch.exp(
+      torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
+  scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+  signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+  signal = F.pad(signal, [0, 0, 0, channels % 2])
+  signal = signal.view(1, channels, length)
+  return signal
+
+
+def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
+  b, channels, length = x.size()
+  signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+  return x + signal.to(dtype=x.dtype, device=x.device)
+
+
+def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
+  b, channels, length = x.size()
+  signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+  return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
+
+
+def subsequent_mask(length):
+  mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+  return mask
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+  n_channels_int = n_channels[0]
+  in_act = input_a + input_b
+  t_act = torch.tanh(in_act[:, :n_channels_int, :])
+  s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+  acts = t_act * s_act
+  return acts
+
+
+def convert_pad_shape(pad_shape):
+  l = pad_shape[::-1]
+  pad_shape = [item for sublist in l for item in sublist]
+  return pad_shape
+
+
+def shift_1d(x):
+  x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
+  return x
+
+
+def sequence_mask(length, max_length=None):
+  if max_length is None:
+    max_length = length.max()
+  x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+  return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def generate_path(duration, mask):
+  """
+  duration: [b, 1, t_x]
+  mask: [b, 1, t_y, t_x]
+  """
+  device = duration.device
+  
+  b, _, t_y, t_x = mask.shape
+  cum_duration = torch.cumsum(duration, -1)
+  
+  cum_duration_flat = cum_duration.view(b * t_x)
+  path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+  path = path.view(b, t_x, t_y)
+  path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
+  path = path.unsqueeze(1).transpose(2,3) * mask
+  return path
+
+
+def clip_grad_value_(parameters, clip_value, norm_type=2):
+  if isinstance(parameters, torch.Tensor):
+    parameters = [parameters]
+  parameters = list(filter(lambda p: p.grad is not None, parameters))
+  norm_type = float(norm_type)
+  if clip_value is not None:
+    clip_value = float(clip_value)
+
+  total_norm = 0
+  for p in parameters:
+    param_norm = p.grad.data.norm(norm_type)
+    total_norm += param_norm.item() ** norm_type
+    if clip_value is not None:
+      p.grad.data.clamp_(min=-clip_value, max=clip_value)
+  total_norm = total_norm ** (1. / norm_type)
+  return total_norm

configs/vtubers.json

@@ -0,0 +1,53 @@
+{
+  "train": {
+    "log_interval": 200,
+    "eval_interval": 400,
+    "seed": 1234,
+    "epochs": 10000,
+    "learning_rate": 2e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 8,
+    "fp16_run": false,
+    "lr_decay": 0.999875,
+    "segment_size": 8192,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "training_files":"filelists/train.txt.cleaned",
+    "validation_files":"filelists/val.txt.cleaned",
+    "text_cleaners":["japanese_cleaners"],
+    "max_wav_value": 32768.0,
+    "sampling_rate": 22050,
+    "filter_length": 1024,
+    "hop_length": 256,
+    "win_length": 1024,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null,
+    "add_blank": true,
+    "n_speakers": 7,
+    "cleaned_text": true
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0.1,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [8,8,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4],
+    "n_layers_q": 3,
+    "use_spectral_norm": false,
+    "gin_channels": 256
+  }
+}

data_utils.py

@@ -0,0 +1,261 @@
+import time
+import os
+import random
+import numpy as np
+import torch
+import torch.utils.data
+
+import commons 
+from mel_processing import spectrogram_torch
+from utils import load_wav_to_torch, load_filepaths_and_text
+from text import text_to_sequence, cleaned_text_to_sequence
+
+"""Multi speaker version"""
+class TextAudioSpeakerLoader(torch.utils.data.Dataset):
+    """
+        1) loads audio, speaker_id, text pairs
+        2) normalizes text and converts them to sequences of integers
+        3) computes spectrograms from audio files.
+    """
+    def __init__(self, audiopaths_sid_text, hparams):
+        self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
+        self.text_cleaners = hparams.text_cleaners
+        self.max_wav_value = hparams.max_wav_value
+        self.sampling_rate = hparams.sampling_rate
+        self.filter_length  = hparams.filter_length
+        self.hop_length     = hparams.hop_length
+        self.win_length     = hparams.win_length
+        self.sampling_rate  = hparams.sampling_rate
+
+        self.cleaned_text = getattr(hparams, "cleaned_text", False)
+
+        self.add_blank = hparams.add_blank
+        self.min_text_len = getattr(hparams, "min_text_len", 1)
+        self.max_text_len = getattr(hparams, "max_text_len", 190)
+
+        random.seed(1234)
+        random.shuffle(self.audiopaths_sid_text)
+        self._filter()
+
+    def _filter(self):
+        """
+        Filter text & store spec lengths
+        """
+        # Store spectrogram lengths for Bucketing
+        # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
+        # spec_length = wav_length // hop_length
+
+        audiopaths_sid_text_new = []
+        lengths = []
+        for audiopath, sid, text in self.audiopaths_sid_text:
+            if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
+                audiopaths_sid_text_new.append([audiopath, sid, text])
+                lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
+        self.audiopaths_sid_text = audiopaths_sid_text_new
+        self.lengths = lengths
+
+    def get_audio_text_speaker_pair(self, audiopath_sid_text):
+        # separate filename, speaker_id and text
+        audiopath, sid, text = audiopath_sid_text[0], audiopath_sid_text[1], audiopath_sid_text[2]
+        text = self.get_text(text)
+        spec, wav = self.get_audio(audiopath)
+        sid = self.get_sid(sid)
+        emo = torch.FloatTensor(np.load(audiopath+".emo.npy"))
+        return (text, spec, wav, sid, emo)
+
+    def get_audio(self, filename):
+        audio, sampling_rate = load_wav_to_torch(filename)
+        if sampling_rate != self.sampling_rate:
+            raise ValueError("{} {} SR doesn't match target {} SR".format(
+                sampling_rate, self.sampling_rate))
+        audio_norm = audio / self.max_wav_value
+        audio_norm = audio_norm.unsqueeze(0)
+        spec_filename = filename.replace(".wav", ".spec.pt")
+        if os.path.exists(spec_filename):
+            spec = torch.load(spec_filename)
+        else:
+            spec = spectrogram_torch(audio_norm, self.filter_length,
+                self.sampling_rate, self.hop_length, self.win_length,
+                center=False)
+            spec = torch.squeeze(spec, 0)
+            torch.save(spec, spec_filename)
+        return spec, audio_norm
+
+    def get_text(self, text):
+        if self.cleaned_text:
+            text_norm = cleaned_text_to_sequence(text)
+        else:
+            text_norm = text_to_sequence(text, self.text_cleaners)
+        if self.add_blank:
+            text_norm = commons.intersperse(text_norm, 0)
+        text_norm = torch.LongTensor(text_norm)
+        return text_norm
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def __getitem__(self, index):
+        return self.get_audio_text_speaker_pair(self.audiopaths_sid_text[index])
+
+    def __len__(self):
+        return len(self.audiopaths_sid_text)
+
+
+class TextAudioSpeakerCollate():
+    """ Zero-pads model inputs and targets
+    """
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        """Collate's training batch from normalized text, audio and speaker identities
+        PARAMS
+        ------
+        batch: [text_normalized, spec_normalized, wav_normalized, sid]
+        """
+        # Right zero-pad all one-hot text sequences to max input length
+        _, ids_sorted_decreasing = torch.sort(
+            torch.LongTensor([x[1].size(1) for x in batch]),
+            dim=0, descending=True)
+
+        max_text_len = max([len(x[0]) for x in batch])
+        max_spec_len = max([x[1].size(1) for x in batch])
+        max_wav_len = max([x[2].size(1) for x in batch])
+
+        text_lengths = torch.LongTensor(len(batch))
+        spec_lengths = torch.LongTensor(len(batch))
+        wav_lengths = torch.LongTensor(len(batch))
+        sid = torch.LongTensor(len(batch))
+
+        text_padded = torch.LongTensor(len(batch), max_text_len)
+        spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
+        wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
+        emo = torch.FloatTensor(len(batch), 1024)
+
+        text_padded.zero_()
+        spec_padded.zero_()
+        wav_padded.zero_()
+        emo.zero_()
+
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            text = row[0]
+            text_padded[i, :text.size(0)] = text
+            text_lengths[i] = text.size(0)
+
+            spec = row[1]
+            spec_padded[i, :, :spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wav = row[2]
+            wav_padded[i, :, :wav.size(1)] = wav
+            wav_lengths[i] = wav.size(1)
+
+            sid[i] = row[3]
+
+            emo[i, :] = row[4]
+
+        if self.return_ids:
+            return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, ids_sorted_decreasing
+        return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid,emo
+
+
+class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
+    """
+    Maintain similar input lengths in a batch.
+    Length groups are specified by boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
+  
+    It removes samples which are not included in the boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
+    """
+    def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True):
+        super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
+        self.lengths = dataset.lengths
+        self.batch_size = batch_size
+        self.boundaries = boundaries
+  
+        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        self.total_size = sum(self.num_samples_per_bucket)
+        self.num_samples = self.total_size // self.num_replicas
+  
+    def _create_buckets(self):
+        buckets = [[] for _ in range(len(self.boundaries) - 1)]
+        for i in range(len(self.lengths)):
+            length = self.lengths[i]
+            idx_bucket = self._bisect(length)
+            if idx_bucket != -1:
+                buckets[idx_bucket].append(i)
+  
+        for i in range(len(buckets) - 1, 0, -1):
+            if len(buckets[i]) == 0:
+                buckets.pop(i)
+                self.boundaries.pop(i+1)
+  
+        num_samples_per_bucket = []
+        for i in range(len(buckets)):
+            len_bucket = len(buckets[i])
+            total_batch_size = self.num_replicas * self.batch_size
+            rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size
+            num_samples_per_bucket.append(len_bucket + rem)
+        return buckets, num_samples_per_bucket
+  
+    def __iter__(self):
+      # deterministically shuffle based on epoch
+      g = torch.Generator()
+      g.manual_seed(self.epoch)
+  
+      indices = []
+      if self.shuffle:
+          for bucket in self.buckets:
+              indices.append(torch.randperm(len(bucket), generator=g).tolist())
+      else:
+          for bucket in self.buckets:
+              indices.append(list(range(len(bucket))))
+  
+      batches = []
+      for i in range(len(self.buckets)):
+          bucket = self.buckets[i]
+          len_bucket = len(bucket)
+          ids_bucket = indices[i]
+          num_samples_bucket = self.num_samples_per_bucket[i]
+  
+          # add extra samples to make it evenly divisible
+          rem = num_samples_bucket - len_bucket
+          ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
+  
+          # subsample
+          ids_bucket = ids_bucket[self.rank::self.num_replicas]
+  
+          # batching
+          for j in range(len(ids_bucket) // self.batch_size):
+              batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]]
+              batches.append(batch)
+  
+      if self.shuffle:
+          batch_ids = torch.randperm(len(batches), generator=g).tolist()
+          batches = [batches[i] for i in batch_ids]
+      self.batches = batches
+  
+      assert len(self.batches) * self.batch_size == self.num_samples
+      return iter(self.batches)
+  
+    def _bisect(self, x, lo=0, hi=None):
+      if hi is None:
+          hi = len(self.boundaries) - 1
+  
+      if hi > lo:
+          mid = (hi + lo) // 2
+          if self.boundaries[mid] < x and x <= self.boundaries[mid+1]:
+              return mid
+          elif x <= self.boundaries[mid]:
+              return self._bisect(x, lo, mid)
+          else:
+              return self._bisect(x, mid + 1, hi)
+      else:
+          return -1
+
+    def __len__(self):
+        return self.num_samples // self.batch_size

emotion_extract.py

@@ -0,0 +1,112 @@
+import torch
+import torch.nn as nn
+from transformers import Wav2Vec2Processor
+from transformers.models.wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2Model,
+    Wav2Vec2PreTrainedModel,
+)
+import os
+import librosa
+import numpy as np
+
+
+class RegressionHead(nn.Module):
+    r"""Classification head."""
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.final_dropout)
+        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+
+        return x
+
+
+class EmotionModel(Wav2Vec2PreTrainedModel):
+    r"""Speech emotion classifier."""
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.config = config
+        self.wav2vec2 = Wav2Vec2Model(config)
+        self.classifier = RegressionHead(config)
+        self.init_weights()
+
+    def forward(
+            self,
+            input_values,
+    ):
+        outputs = self.wav2vec2(input_values)
+        hidden_states = outputs[0]
+        hidden_states = torch.mean(hidden_states, dim=1)
+        logits = self.classifier(hidden_states)
+
+        return hidden_states, logits
+
+
+# load model from hub
+device = 'cuda' if torch.cuda.is_available() else "cpu"
+model_name = 'audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'
+processor = Wav2Vec2Processor.from_pretrained(model_name)
+model = EmotionModel.from_pretrained(model_name).to(device)
+
+
+def process_func(
+        x: np.ndarray,
+        sampling_rate: int,
+        embeddings: bool = False,
+) -> np.ndarray:
+    r"""Predict emotions or extract embeddings from raw audio signal."""
+
+    # run through processor to normalize signal
+    # always returns a batch, so we just get the first entry
+    # then we put it on the device
+    y = processor(x, sampling_rate=sampling_rate)
+    y = y['input_values'][0]
+    y = torch.from_numpy(y).to(device)
+
+    # run through model
+    with torch.no_grad():
+        y = model(y)[0 if embeddings else 1]
+
+    # convert to numpy
+    y = y.detach().cpu().numpy()
+
+    return y
+#
+#
+# def disp(rootpath, wavname):
+#     wav, sr = librosa.load(f"{rootpath}/{wavname}", 16000)
+#     display(ipd.Audio(wav, rate=sr))
+
+rootpath = "dataset/nene"
+embs = []
+wavnames = []
+def extract_dir(path):
+    rootpath = path
+    for idx, wavname in enumerate(os.listdir(rootpath)):
+        wav, sr =librosa.load(f"{rootpath}/{wavname}", 16000)
+        emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
+        embs.append(emb)
+        wavnames.append(wavname)
+        np.save(f"{rootpath}/{wavname}.emo.npy", emb.squeeze(0))
+        print(idx, wavname)
+
+def extract_wav(path):
+    wav, sr = librosa.load(path, 16000)
+    emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
+    return emb
+
+if __name__ == '__main__':
+    for spk in ["serena", "koni", "nyaru","shanoa", "mana"]:
+        extract_dir(f"dataset/{spk}")

filelists/val.txt

@@ -0,0 +1,63 @@
+dataset/nene/nen001_001.wav|0|はい？呼びました？
+dataset/nene/nen001_002.wav|0|驚かせたならごめんなさい。通りかかったときにちょうど名前が聞こえてきたので
+dataset/nene/nen001_003.wav|0|私に何か用ですか？
+dataset/nene/nen001_004.wav|0|いえ、少し気になっただけですから、別に怒っているわけじゃないです。気にしないで下さい
+dataset/nene/nen001_005.wav|0|それより仮屋さん、例の件ですが――
+dataset/nene/nen001_006.wav|0|喜んでもらえたならなによりです
+dataset/nene/nen001_007.wav|0|また何かあったら、いつでも部室に来て下さい
+dataset/nene/nen001_008.wav|0|大したことじゃないので。それより体調の方はもういいんですか？
+dataset/nene/nen001_009.wav|0|それはよかったです。他に困ったことはありますか？
+dataset/nene/nen001_010.wav|0|何かあったらいつでも話して下さい。学院のことじゃなく、私事に関することでも何でも
+dataset/nene/nen001_011.wav|0|はい、いつでもどうぞ
+dataset/nene/nen001_012.wav|0|保科君も
+dataset/nene/nen001_013.wav|0|もし何か困ったことがあれば、力になりますから
+dataset/nene/nen001_014.wav|0|そうですか？私には、なにか悩み事があるように見えたりしましたけど……
+dataset/nene/nen001_015.wav|0|――なんて、言ってみただけですから、深い意味はありませんよ
+dataset/nene/nen001_016.wav|0|いえ、そういうわけじゃなくって……ただ何となく。そんな気がしただけですから
+dataset/nene/nen001_017.wav|0|あ、いえ、絶対に秘密というわけじゃないので気にしないで下さい
+dataset/nene/nen001_018.wav|0|好奇心だけで来られると困るので、本当に悩んでる人以外には、広めないようにお願いしているだけです
+dataset/nene/nen001_019.wav|0|そんな仰々しい話じゃなく……私は、オカ研に所属しているんです
+dataset/nene/nen001_020.wav|0|はい。そのオカ研です
+dataset/nene/nen001_021.wav|0|そうですよ。実は私たちが入学する前からあった部活なんです。今は部員がいなくて、所属してるのは私一人ですが
+dataset/nene/nen001_022.wav|0|なので最近は、学生会の方からは結構つつかれてて……活動も、発表などを意欲的にしているわけじゃありませんからね
+dataset/nene/nen001_023.wav|0|いえ、私は占いを。オカルトと言っても幅は広いので。それに部には私しかいませんから、結構好きにできるんです
+dataset/nene/nen001_024.wav|0|さすがに白蛇占いはできませんよ
+dataset/nene/nen001_025.wav|0|一応。知ってはいますけど、私にできるのはタロットぐらいです。あくまで趣味程度ですから
+dataset/nene/nen001_026.wav|0|あくまで占いの延長線上のものですから。人生相談なんていうほど大層な物じゃありません
+dataset/nene/nen001_027.wav|0|でも……もし保科君も嫌いでなければ、いつでも部室に来て下さい
+dataset/nene/nen001_028.wav|0|あ、はい。すぐに行きます
+dataset/nene/nen001_029.wav|0|それじゃあ私はこれで
+dataset/nene/nen001_030.wav|0|あの、先生
+dataset/nene/nen001_031.wav|0|はい。気付いたらこんな時間になってしまって
+dataset/nene/nen001_032.wav|0|図書室は……もう誰もいないんですか？
+dataset/nene/nen001_033.wav|0|その前にちょっと調べ物をさせて欲しいんですが、鍵を貸してもらえませんか？
+dataset/nene/nen001_034.wav|0|タロットカードのことで少々。時間はかかりません。５分……は無理かも……でも２０分もあれば終わりますから……お願いします
+dataset/nene/nen001_035.wav|0|わかりました。ありがとう……ございます
+dataset/nene/nen001_036.wav|0|は、はい………気をつけます……ハァ、ハァ……
+dataset/nene/nen001_037.wav|0|……ハァ……ハァ……
+dataset/nene/nen001_038.wav|0|……大丈夫、ですよね……んっ……
+dataset/nene/nen001_039.wav|0|……ハァ……ハァ……
+dataset/nene/nen001_040.wav|0|んっ、んん……
+dataset/nene/nen001_041.wav|0|あ……あっ、んっ……んんン、んッ、んッ、んぅぅッ……
+dataset/nene/nen001_042.wav|0|んっ、ふぅぅ……はぁ、はぁ、あっ、ああぁぁ……ふぁぁぁ……
+dataset/nene/nen001_043.wav|0|はぁ、はぁぁ……ん、ん、んっ……はぁぁ……ぁ、ぁ、ぁ……ぁぁ……ンッ……！
+dataset/nene/nen001_044.wav|0|あぁ、もう……こ、んなの、最低ですッ……はぁ、はぁ……学院内で、おっ、オナニー……をするなんてっ、ん、んんっ
+dataset/nene/nen001_045.wav|0|あっ、ああぁぁ……はぁ、はぁ、はぁぁぁ……ん、んンッッ
+dataset/nene/nen001_046.wav|0|はぁ、はぁ……私、��んてこと……あっ、あぁぁ……でも、気持ちよくて止まりません……ん、んんッ、ふぁ、あ、あぁぁぁ……
+dataset/nene/nen001_047.wav|0|うっ……あ、あ、あっ、あっ、ひあっ、ん、んはっ……はっ、はっ、んんぅぅッ
+dataset/nene/nen001_048.wav|0|はぁ……はぁ……んっ、んんん……んぁ、んぁ、あっ……んっ、ぃぃぃッ
+dataset/nene/nen001_049.wav|0|んっ、んっ、んくっ……ひっ、あっ、ぁっ、ぁっ、んんーーッ……
+dataset/nene/nen001_050.wav|0|はぁぁぁ……はぁ、はぁ……んんっ、んっ、んんん、んぁ……んぁ、ぁ、ぁ、ぁぁぁぁ……ッ！
+dataset/nene/nen001_051.wav|0|だめ、早く、しないと……先生が、戻って、きます……こんなところ、見られたら……んっ、んっ、んぁ、んぁ、あ、あ、あ、あ、あ……
+dataset/nene/nen001_052.wav|0|あ、あ、ん、んんんッ……はっ、はぁ、はぁ、あぁぁもう、本当に、最低ですっ……
+dataset/nene/nen001_053.wav|0|んッ……んはぁッ、はぁ、はぁ……ンン、んぁ、んぁぁ……あ、あ、あぁぁ……
+dataset/nene/nen001_054.wav|0|はぁー、はぁーーぁぁ、気持ちいい……あっ、あぃ、あぃっ、いい……本当に……んん
+dataset/nene/nen001_055.wav|0|早く……こんなこと、早く……先生が、また様子を……見に、きたりしない、内に……はぁ、はぁ、はぁ、はぁ
+dataset/nene/nen001_056.wav|0|はぁ、はぁ、はぁっ、はぁぁぁ……ぁ、ぁ、ぁ……ぁぁ……ぁぁんッ、んっ、んーッ
+dataset/nene/nen001_057.wav|0|んんッ、ん、んはぁぁぁーー……はぁ、はぁ、はぁァン、あッ、あん、あぁぁンッ
+dataset/nene/nen001_058.wav|0|はぁ、はぁ、はぁぁ……ヨダレ、でちゃう……じゅる……んぁっ、はぁ、はぁぁ……はっ、はっ、はっ……じゅる
+dataset/nene/nen001_059.wav|0|こんな、に、気持ちいいなんて、最低です……本当、図書室でオナニーなんて、最低すぎます……でも、でも
+dataset/nene/nen001_060.wav|0|あああぁぁ……今は、止められなくて……じゅる……はぁ、はぁぁ……あぁぁぁあぁ……
+dataset/nene/nen001_061.wav|0|はぁ、はぁ、はぁ……早く……あっ、あっ、あぁぁ……ぁぁぁ、早く、早く
+dataset/nene/nen001_062.wav|0|早く、イきたい、イきたい……このままイきたいぃ……はぁ、はぁ、はぁ、ぁぁ、ぁっ、ぁっ、あっ、あっ、あぁッ！
+dataset/nene/nen001_063.wav|0|あっ、あっ、ああぁぁ……痺れて、きたぁ……だめ、イきそう……んっ、だめじゃなくて、あっ、あっ、早く、このまま……早く早く早く

filelists/val.txt.cleaned

@@ -0,0 +1,63 @@
+dataset/nene/nen001_001.wav|0|ha↓i? yo↑bima↓ʃIta?
+dataset/nene/nen001_002.wav|0|o↑doro↓kasetanara go↑meNnasa↓i. to↑orikaka↓Qta to↓kini ʧo↑odo na↑maega kI↑koete ki↓tanode.
+dataset/nene/nen001_003.wav|0|wa↑taʃini na↑nika↓yoodesUka?
+dataset/nene/nen001_004.wav|0|i↓e, sU↑ko↓ʃI ki↑ni na↓QtadakedesUkara, be↑tsuni o↑ko↓Qte i↑ru wa↓kejanaidesU. ki↑ni ʃi↑na↓ide ku↑dasa↓i.
+dataset/nene/nen001_005.wav|0|so↑reyo↓ri ka↑riyasaN, re↓eno ke↓NdesUga----
+dataset/nene/nen001_006.wav|0|yo↑roko↓Nde mo↑raeta↓nara na↓niyoridesU.
+dataset/nene/nen001_007.wav|0|ma↑ta na↓nika a↓Qtara, i↓tsudemo bu↑ʃItsuni ki↓te ku↑dasa↓i.
+dataset/nene/nen001_008.wav|0|ta↓iʃIta ko↑to↓janainode. so↑reyo↓ri ta↑iʧoono ho↓owa mo↓o i↓i N↓desUka?
+dataset/nene/nen001_009.wav|0|so↑rewa yo↓kaQtadesU. ta↓ni ko↑ma↓Qta ko↑to↓wa a↑rima↓sUka?
+dataset/nene/nen001_010.wav|0|na↓nika a↓Qtara i↓tsudemo ha↑na↓ʃIte ku↑dasa↓i. ga↑kuiNno ko↑to↓janaku, ʃi↓jini ka↑Nsu↓ru ko↑to↓demo na↓nidemo.
+dataset/nene/nen001_011.wav|0|ha↓i, i↓tsudemo do↓ozo.
+dataset/nene/nen001_012.wav|0|ho↓ʃinakuNmo.
+dataset/nene/nen001_013.wav|0|mo↓ʃi na↓nika ko↑ma↓Qta ko↑to↓ga a↑re↓ba, ʧI↑kara↓ni na↑rima↓sUkara.
+dataset/nene/nen001_014.wav|0|so↑odesU↓ka? wa↑taʃiniwa, na↓nika na↑yami↓gotoga a↓ru yo↓oni mi↑eta↓ri ʃi↑ma↓ʃItakedo......
+dataset/nene/nen001_015.wav|0|---- na↓Nte, i↑Qte mi↓tadakedesUkara, fU↑ka↓i i↑miwaarimase↓Nyo.
+dataset/nene/nen001_016.wav|0|i↓e, so↑oyuu wa↓kejanakuQte...... ta↓da na↑Ntona↓ku. so↑Nna ki↑ga ʃI↑ta↓dakedesUkara.
+dataset/nene/nen001_017.wav|0|a, i↓e, ze↑Qtaini hi↑mitsUto i↑u wa↓kejanainode ki↑ni ʃi↑na↓ide ku↑dasa↓i.
+dataset/nene/nen001_018.wav|0|ko↑okIʃiNdakede ko↑rare↓ruto ko↑ma↓runode, ho↑Ntooni na↑ya↓Nde ru↑niNi↓gainiwa, hi↑romenai yo↓oni o↑negai ʃI↑te i↑rudakedesU.
+dataset/nene/nen001_019.wav|0|so↑Nna gyo↑ogyooʃi↓i ha↑naʃi↓janaku...... wa↑taʃiwa, o↑kakeNni ʃo↑zoku ʃI↑te i↑ru N↓desU.
+dataset/nene/nen001_020.wav|0|ha↓i. so↑no o↑kakeNde↓sU.
+dataset/nene/nen001_021.wav|0|so↑odesUyo. ji↑tsu↓wa wa↑taʃi↓taʧiga nyu↑ugakU su↑ru ma↓ekara a↓Qta bu↑katsuna N↓desU. i↓mawa bu↑iNga i↑nakUte, ʃo↑zoku ʃI↑te↓ru no↑wa wa↑taʃI hi↑to↓ridesUga.
+dataset/nene/nen001_022.wav|0|na↑node sa↑ikiNwa, ga↑kUsee↓kaino ho↓okarawa ke↓Qkoo tsU↑tsu↓karetete...... ka↑tsudoomo, ha↑Qpyoona↓doo i↑yoku↓tekini ʃI↑te i↑ru wa↑kejaarimase↓Nkarane.
+dataset/nene/nen001_023.wav|0|i↓e, wa↑taʃiwa u↑ranaio. o↑karutoto i↑Qtemo ha↑bawa hi↑ro↓inode. so↑reni bu↓niwa wa↑taʃIʃi↓ka i↑mase↓Nkara, ke↓Qkoo sU↑ki↓ni de↑ki↓ru N↓desU.
+dataset/nene/nen001_024.wav|0|sa↑sugani ʃi↑rohebiu↓ranaiwa de↑kimase↓Nyo.
+dataset/nene/nen001_025.wav|0|i↑ʧioo. ʃi↑Qte ha↑ima↓sUkedo, wa↑taʃini de↑ki↓ru no↑wa ta↑roQtogu↓raidesU. a↑ku↓made ʃu↑mite↓edodesUkara.
+dataset/nene/nen001_026.wav|0|a↑ku↓made u↑ranaino e↑NʧooseNjoono mo↑no↓desUkara. ji↑Nseeso↓odaNnaNte i↑uhodo ta↓isoona mo↑nojaarimase↓N.
+dataset/nene/nen001_027.wav|0|de↓mo...... mo↓ʃI ho↓ʃinakuNmo ki↑raidenakere↓ba, i↓tsudemo bu↑ʃItsuni ki↓te ku↑dasa↓i.
+dataset/nene/nen001_028.wav|0|a, ha↓i. su↓guni i↑kima↓sU.
+dataset/nene/nen001_029.wav|0|so↑reja↓a wa↑taʃiwa ko↑rede.
+dataset/nene/nen001_030.wav|0|a↑no, se↑Nse↓e.
+dataset/nene/nen001_031.wav|0|ha↓i. ki↑zu↓itara ko↑Nna ji↑kaNni na↓Qte ʃi↑ma↓Qte.
+dataset/nene/nen001_032.wav|0|to↑ʃo↓ʃItsuwa...... mo↓o da↓remo i↑nai N↓desUka?
+dataset/nene/nen001_033.wav|0|so↑no ma↓eni ʧo↓Qto ʃi↑rabebutsuo sa↑setehoʃii N↓desUga, ka↑gi↓o ka↑ʃIte mo↑raemase↓Nka?
+dataset/nene/nen001_034.wav|0|ta↑roQtoka↓adono ko↑to↓de ʃo↓oʃoo. ji↑kaNwa ka↑karimase↓N. go↓fuN...... w a mu↓rikamo...... de↓mo ni↑juQ↓puNmo a↑re↓ba o↑warima↓sUkara...... o↑negai ʃi↑ma↓sU.
+dataset/nene/nen001_035.wav|0|wa↑karima↓ʃIta. a↑ri↓gatoo...... go↑zaima↓sU.
+dataset/nene/nen001_036.wav|0|w a, ha↓i......... ki↑o tsU↑kema↓sU...... ha↓a, ha↓a......
+dataset/nene/nen001_037.wav|0|...... ha↓a...... ha↓a......
+dataset/nene/nen001_038.wav|0|...... da↑ijo↓obu, de↓sUyone...... N↓Q......
+dataset/nene/nen001_039.wav|0|...... ha↓a...... ha↓a......
+dataset/nene/nen001_040.wav|0|N↓Q, N↓N......
+dataset/nene/nen001_041.wav|0|a...... a↓Q, N↓Q...... N↓N N, N↓Q, N↓Q, N↓uuQ......
+dataset/nene/nen001_042.wav|0|N↓Q, fu↓uu...... ha↓a, ha↓a, a↓Q, a↓aaa...... fa↓aa......
+dataset/nene/nen001_043.wav|0|ha↓a, ha↓aa...... N, N, N↓Q...... ha↓aa...... a, a, a...... a↓a...... N↓Q......!
+dataset/nene/nen001_044.wav|0|a↓a, mo↓o...... k o, N↑na n o, sa↑iteede su↓Q...... ha↓a, ha↓a...... ga↑kuiN↓naide, o↑Q, o↓nanii...... o su↑ru↓naNteQ, N, N↓NQ.
+dataset/nene/nen001_045.wav|0|a↓Q, a↓aaa...... ha↓a, ha↓a, ha↓aaa...... N, N↓NQQ.
+dataset/nene/nen001_046.wav|0|ha↓a, ha↓a...... wa↑taʃi, na↓Nte ko↑to...... a↓Q, a↓aa...... de↓mo, ki↑moʧiyo↓kUte to↑marimase↓N...... N, N↓NQ, f a, a, a↓aaa......
+dataset/nene/nen001_047.wav|0|u↓Q...... a, a, a↓Q, a↓Q, hi↓aQ, N, N↓haQ...... ha↓Q, ha↓Q, N↓NuuQ.
+dataset/nene/nen001_048.wav|0|ha↓a...... ha↓a...... N↓Q, N↓NN...... N↓a, N↓a, a↓Q...... N↓Q, i↓iiQ.
+dataset/nene/nen001_049.wav|0|N↓Q, N↓Q, N ku↓Q...... hi↓Q, a↓Q, a↓Q, a↓Q, N↓NNNQ......
+dataset/nene/nen001_050.wav|0|ha↓aaa...... ha↓a, ha↓a...... N↓NQ, N↓Q, N↓NN, N↓a...... N↓a, a, a, a↓aaa...... Q!
+dataset/nene/nen001_051.wav|0|da↑me, ha↓yaku, ʃi↑naito...... se↑Nse↓ega, mo↑do↓Qte, ki↑ma↓sU...... ko↑Nna to↑koro, mi↓raretara...... N↓Q, N↓Q, N↓a, N↓a, a, a, a, a, a......
+dataset/nene/nen001_052.wav|0|a, a, N, N↓NN Q...... ha↓Q, ha↓a, ha↓a, a↓aa mo↓o, ho↑Ntooni, sa↑iteede su↓Q......
+dataset/nene/nen001_053.wav|0|N↓Q...... N↓haaQ, ha↓a, ha↓a...... N↓N, N↓a, N↓aa...... a, a, a↓aa......
+dataset/nene/nen001_054.wav|0|ha↓aa, ha↓aaaaa, ki↑moʧii↓i...... a↓Q, a↓i, a↓iQ, i↓i...... ho↑Ntooni...... N↓N.
+dataset/nene/nen001_055.wav|0|ha↓yaku...... ko↑Nna ko↑to, ha↓yaku...... se↑Nse↓ega, ma↑ta yo↑osuo...... mi↑ni, kI↑tariʃi↓nai, u↑ʧini...... ha↓a, ha↓a, ha↓a, ha↓a.
+dataset/nene/nen001_056.wav|0|ha↓a, ha↓a, ha↓aQ, ha↓aaa...... a, a, a...... a↓a...... a↓aNQ, N↓Q, N↓NQ.
+dataset/nene/nen001_057.wav|0|N↓NQ, N, N↓haaaaaa...... ha↓a, ha↓a, ha↓aaN, a↓Q, a↑N, a↓aaNQ.
+dataset/nene/nen001_058.wav|0|ha↓a, ha↓a, ha↓aa...... yo↑dare, de↑ʧau...... ju↑ru...... N↓aQ, ha↓a, ha↓aa...... ha↓Q, ha↓Q, ha↓Q...... ju↑ru.
+dataset/nene/nen001_059.wav|0|ko↑Nna, n i, ki↑moʧii↓inaNte, sa↑iteede↓sU...... ho↑Ntoo, to↑ʃo↓ʃItsude o↓naniinaNte, sa↑itee su↑gima↓sU...... de↓mo, de↓mo.
+dataset/nene/nen001_060.wav|0|a↑a↓aaa...... i↓mawa, to↑merarenakUte...... ju↑ru...... ha↓a, ha↓aa...... a↑aaa↓aa......
+dataset/nene/nen001_061.wav|0|ha↓a, ha↓a, ha↓a...... ha↓yaku...... a↓Q, a↓Q, a↓aa...... a↓aa, ha↓yaku, ha↓yaku.
+dataset/nene/nen001_062.wav|0|ha↓yaku, i↑ki↓tai, i↑ki↓tai...... ko↑no ma↑maiki↓taii...... ha↓a, ha↓a, ha↓a, a↓a, a↓Q, a↓Q, a↓Q, a↓Q, a↓aQ!
+dataset/nene/nen001_063.wav|0|a↓Q, a↓Q, a↓aaa...... ʃi↑bire↓te, ki↓ta a...... da↑me, i↓kIsou...... N↓Q, da↑me↓janakUte, a↓Q, a↓Q, ha↓yaku, ko↑no ma↑ma...... ha↓yakUhayakUhayaku.

losses.py

@@ -0,0 +1,61 @@
+import torch 
+from torch.nn import functional as F
+
+import commons
+
+
+def feature_loss(fmap_r, fmap_g):
+  loss = 0
+  for dr, dg in zip(fmap_r, fmap_g):
+    for rl, gl in zip(dr, dg):
+      rl = rl.float().detach()
+      gl = gl.float()
+      loss += torch.mean(torch.abs(rl - gl))
+
+  return loss * 2 
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+  loss = 0
+  r_losses = []
+  g_losses = []
+  for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+    dr = dr.float()
+    dg = dg.float()
+    r_loss = torch.mean((1-dr)**2)
+    g_loss = torch.mean(dg**2)
+    loss += (r_loss + g_loss)
+    r_losses.append(r_loss.item())
+    g_losses.append(g_loss.item())
+
+  return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+  loss = 0
+  gen_losses = []
+  for dg in disc_outputs:
+    dg = dg.float()
+    l = torch.mean((1-dg)**2)
+    gen_losses.append(l)
+    loss += l
+
+  return loss, gen_losses
+
+
+def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
+  """
+  z_p, logs_q: [b, h, t_t]
+  m_p, logs_p: [b, h, t_t]
+  """
+  z_p = z_p.float()
+  logs_q = logs_q.float()
+  m_p = m_p.float()
+  logs_p = logs_p.float()
+  z_mask = z_mask.float()
+
+  kl = logs_p - logs_q - 0.5
+  kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
+  kl = torch.sum(kl * z_mask)
+  l = kl / torch.sum(z_mask)
+  return l

mel_processing.py

@@ -0,0 +1,119 @@
+import math
+import os
+import random
+import torch
+from torch import nn
+import torch.nn.functional as F
+import torch.utils.data
+import numpy as np
+
+import logging
+
+numba_logger = logging.getLogger('numba')
+numba_logger.setLevel(logging.WARNING)
+import warnings
+warnings.filterwarnings('ignore')
+import librosa
+import librosa.util as librosa_util
+from librosa.util import normalize, pad_center, tiny
+from scipy.signal import get_window
+from scipy.io.wavfile import read
+from librosa.filters import mel as librosa_mel_fn
+
+MAX_WAV_VALUE = 32768.0
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    """
+    PARAMS
+    ------
+    C: compression factor
+    """
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+    """
+    PARAMS
+    ------
+    C: compression factor used to compress
+    """
+    return torch.exp(x) / C
+
+
+def spectral_normalize_torch(magnitudes):
+    output = dynamic_range_compression_torch(magnitudes)
+    return output
+
+
+def spectral_de_normalize_torch(magnitudes):
+    output = dynamic_range_decompression_torch(magnitudes)
+    return output
+
+
+mel_basis = {}
+hann_window = {}
+
+
+def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
+    if torch.min(y) < -1.:
+        print('min value is ', torch.min(y))
+    if torch.max(y) > 1.:
+        print('max value is ', torch.max(y))
+
+    global hann_window
+    dtype_device = str(y.dtype) + '_' + str(y.device)
+    wnsize_dtype_device = str(win_size) + '_' + dtype_device
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
+
+    y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
+    y = y.squeeze(1)
+
+    spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
+                      center=center, pad_mode='reflect', normalized=False, onesided=True)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+    return spec
+
+
+def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
+    global mel_basis
+    dtype_device = str(spec.dtype) + '_' + str(spec.device)
+    fmax_dtype_device = str(fmax) + '_' + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device)
+    spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    spec = spectral_normalize_torch(spec)
+    return spec
+
+
+def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
+    if torch.min(y) < -1.:
+        print('min value is ', torch.min(y))
+    if torch.max(y) > 1.:
+        print('max value is ', torch.max(y))
+
+    global mel_basis, hann_window
+    dtype_device = str(y.dtype) + '_' + str(y.device)
+    fmax_dtype_device = str(fmax) + '_' + dtype_device
+    wnsize_dtype_device = str(win_size) + '_' + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device)
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
+
+    y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
+    y = y.squeeze(1)
+
+    spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
+                      center=center, pad_mode='reflect', normalized=False, onesided=True)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+
+    spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    spec = spectral_normalize_torch(spec)
+
+    return spec