Spaces:
Sleeping
Sleeping
init
Browse files- G_49200.pth +3 -0
- LICENSE +21 -0
- all_emotions.npy +3 -0
- app.py +111 -0
- attentions.py +303 -0
- commons.py +161 -0
- configs/vtubers.json +53 -0
- data_utils.py +261 -0
- emotion_extract.py +112 -0
- filelists/train.txt +0 -0
- filelists/train.txt.cleaned +0 -0
- filelists/val.txt +63 -0
- filelists/val.txt.cleaned +63 -0
- losses.py +61 -0
- mel_processing.py +119 -0
- models.py +537 -0
- modules.py +390 -0
- monotonic_align/__init__.py +19 -0
- monotonic_align/core.pyx +42 -0
- monotonic_align/setup.py +9 -0
- preprocess.py +25 -0
- requirements.txt +13 -0
- resources/fig_1a.png +0 -0
- resources/fig_1b.png +0 -0
- resources/training.png +0 -0
- text/LICENSE +19 -0
- text/__init__.py +56 -0
- text/cleaners.py +178 -0
- text/japanese.py +153 -0
- text/mandarin.py +328 -0
- text/symbols.py +69 -0
- train_ms.py +296 -0
- transforms.py +193 -0
- utils.py +267 -0
G_49200.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:da931601660595a5ab841b25668faf97ee66e10d3ae5da03f69c5e61f28476fd
|
3 |
+
size 479164585
|
LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
MIT License
|
2 |
+
|
3 |
+
Copyright (c) 2021 Jaehyeon Kim
|
4 |
+
|
5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6 |
+
of this software and associated documentation files (the "Software"), to deal
|
7 |
+
in the Software without restriction, including without limitation the rights
|
8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9 |
+
copies of the Software, and to permit persons to whom the Software is
|
10 |
+
furnished to do so, subject to the following conditions:
|
11 |
+
|
12 |
+
The above copyright notice and this permission notice shall be included in all
|
13 |
+
copies or substantial portions of the Software.
|
14 |
+
|
15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21 |
+
SOFTWARE.
|
all_emotions.npy
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:48e81667f1fc4ce2b2eaed80fadd0871e1ddfc8933767915954c39ac854d5724
|
3 |
+
size 22356096
|
app.py
ADDED
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import torch
|
3 |
+
import commons
|
4 |
+
import utils
|
5 |
+
from models import SynthesizerTrn
|
6 |
+
from text.symbols import symbols
|
7 |
+
from text import text_to_sequence
|
8 |
+
import numpy as np
|
9 |
+
|
10 |
+
|
11 |
+
def get_text(text, hps):
|
12 |
+
text_norm = text_to_sequence(text, hps.data.text_cleaners)
|
13 |
+
if hps.data.add_blank:
|
14 |
+
text_norm = commons.intersperse(text_norm, 0)
|
15 |
+
text_norm = torch.LongTensor(text_norm)
|
16 |
+
return text_norm
|
17 |
+
hps = utils.get_hparams_from_file("./configs/vtubers.json")
|
18 |
+
net_g = SynthesizerTrn(
|
19 |
+
len(symbols),
|
20 |
+
hps.data.filter_length // 2 + 1,
|
21 |
+
hps.train.segment_size // hps.data.hop_length,
|
22 |
+
n_speakers=hps.data.n_speakers,
|
23 |
+
**hps.model)
|
24 |
+
_ = net_g.eval()
|
25 |
+
|
26 |
+
_ = utils.load_checkpoint("./G_49200.pth", net_g, None)
|
27 |
+
all_emotions = np.load("all_emotions.npy")
|
28 |
+
emotion_dict = {
|
29 |
+
"小声": 2077,
|
30 |
+
"激动": 111,
|
31 |
+
"平静1": 434,
|
32 |
+
"平静2": 3554
|
33 |
+
}
|
34 |
+
import random
|
35 |
+
def tts(txt, emotion):
|
36 |
+
stn_tst = get_text(txt, hps)
|
37 |
+
randsample = None
|
38 |
+
with torch.no_grad():
|
39 |
+
x_tst = stn_tst.unsqueeze(0)
|
40 |
+
x_tst_lengths = torch.LongTensor([stn_tst.size(0)])
|
41 |
+
sid = torch.LongTensor([0])
|
42 |
+
if type(emotion) ==int:
|
43 |
+
emo = torch.FloatTensor(all_emotions[emotion]).unsqueeze(0)
|
44 |
+
elif emotion == "random":
|
45 |
+
emo = torch.randn([1,1024])
|
46 |
+
elif emotion == "random_sample":
|
47 |
+
randint = random.randint(0, all_emotions.shape[0])
|
48 |
+
emo = torch.FloatTensor(all_emotions[randint]).unsqueeze(0)
|
49 |
+
randsample = randint
|
50 |
+
elif emotion.endswith("wav"):
|
51 |
+
import emotion_extract
|
52 |
+
emo = torch.FloatTensor(emotion_extract.extract_wav(emotion))
|
53 |
+
else:
|
54 |
+
emo = torch.FloatTensor(all_emotions[emotion_dict[emotion]]).unsqueeze(0)
|
55 |
+
|
56 |
+
audio = net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=0.667, noise_scale_w=0.8, length_scale=1, emo=emo)[0][0,0].data.float().numpy()
|
57 |
+
return audio, randsample
|
58 |
+
|
59 |
+
|
60 |
+
def tts1(text, emotion):
|
61 |
+
if len(text) > 150:
|
62 |
+
return "Error: Text is too long", None
|
63 |
+
audio, _ = tts(text, emotion)
|
64 |
+
return "Success", (hps.data.sampling_rate, audio)
|
65 |
+
|
66 |
+
def tts2(text):
|
67 |
+
if len(text) > 150:
|
68 |
+
return "Error: Text is too long", None
|
69 |
+
audio, randsample = tts(text, "random_sample")
|
70 |
+
|
71 |
+
return str(randsample), (hps.data.sampling_rate, audio)
|
72 |
+
|
73 |
+
def tts3(text, sample):
|
74 |
+
if len(text) > 150:
|
75 |
+
return "Error: Text is too long", None
|
76 |
+
try:
|
77 |
+
audio, _ = tts(text, int(sample))
|
78 |
+
return "Success", (hps.data.sampling_rate, audio)
|
79 |
+
except:
|
80 |
+
return "输入参数不为整数或其他错误"
|
81 |
+
|
82 |
+
app = gr.Blocks()
|
83 |
+
with app:
|
84 |
+
with gr.Tabs():
|
85 |
+
with gr.TabItem("使用预制情感合成"):
|
86 |
+
tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
|
87 |
+
tts_input2 = gr.Dropdown(label="情感", choices=list(emotion_dict.keys()), value="平静1")
|
88 |
+
tts_submit = gr.Button("合成音频", variant="primary")
|
89 |
+
tts_output1 = gr.Textbox(label="Message")
|
90 |
+
tts_output2 = gr.Audio(label="Output")
|
91 |
+
tts_submit.click(tts1, [tts_input1, tts_input2], [tts_output1, tts_output2])
|
92 |
+
with gr.TabItem("随机抽取训练集样本作为情感参数"):
|
93 |
+
tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
|
94 |
+
tts_submit = gr.Button("合成音频", variant="primary")
|
95 |
+
tts_output1 = gr.Textbox(label="随机样本id(可用于第三个tab中合成)")
|
96 |
+
tts_output2 = gr.Audio(label="Output")
|
97 |
+
tts_submit.click(tts2, [tts_input1], [tts_output1, tts_output2])
|
98 |
+
|
99 |
+
with gr.TabItem("使用情感样本id作为情感参数"):
|
100 |
+
|
101 |
+
tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
|
102 |
+
tts_input2 = gr.Number(label="情感样本id", value=2004)
|
103 |
+
tts_submit = gr.Button("合成音频", variant="primary")
|
104 |
+
tts_output1 = gr.Textbox(label="Message")
|
105 |
+
tts_output2 = gr.Audio(label="Output")
|
106 |
+
tts_submit.click(tts3, [tts_input1, tts_input2], [tts_output1, tts_output2])
|
107 |
+
|
108 |
+
with gr.TabItem("使用参考音频作为情感参数"):
|
109 |
+
tts_input1 = gr.TextArea(label="text", value="暂未实现")
|
110 |
+
|
111 |
+
app.launch()
|
attentions.py
ADDED
@@ -0,0 +1,303 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import copy
|
2 |
+
import math
|
3 |
+
import numpy as np
|
4 |
+
import torch
|
5 |
+
from torch import nn
|
6 |
+
from torch.nn import functional as F
|
7 |
+
|
8 |
+
import commons
|
9 |
+
import modules
|
10 |
+
from modules import LayerNorm
|
11 |
+
|
12 |
+
|
13 |
+
class Encoder(nn.Module):
|
14 |
+
def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
|
15 |
+
super().__init__()
|
16 |
+
self.hidden_channels = hidden_channels
|
17 |
+
self.filter_channels = filter_channels
|
18 |
+
self.n_heads = n_heads
|
19 |
+
self.n_layers = n_layers
|
20 |
+
self.kernel_size = kernel_size
|
21 |
+
self.p_dropout = p_dropout
|
22 |
+
self.window_size = window_size
|
23 |
+
|
24 |
+
self.drop = nn.Dropout(p_dropout)
|
25 |
+
self.attn_layers = nn.ModuleList()
|
26 |
+
self.norm_layers_1 = nn.ModuleList()
|
27 |
+
self.ffn_layers = nn.ModuleList()
|
28 |
+
self.norm_layers_2 = nn.ModuleList()
|
29 |
+
for i in range(self.n_layers):
|
30 |
+
self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
|
31 |
+
self.norm_layers_1.append(LayerNorm(hidden_channels))
|
32 |
+
self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
|
33 |
+
self.norm_layers_2.append(LayerNorm(hidden_channels))
|
34 |
+
|
35 |
+
def forward(self, x, x_mask):
|
36 |
+
attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
|
37 |
+
x = x * x_mask
|
38 |
+
for i in range(self.n_layers):
|
39 |
+
y = self.attn_layers[i](x, x, attn_mask)
|
40 |
+
y = self.drop(y)
|
41 |
+
x = self.norm_layers_1[i](x + y)
|
42 |
+
|
43 |
+
y = self.ffn_layers[i](x, x_mask)
|
44 |
+
y = self.drop(y)
|
45 |
+
x = self.norm_layers_2[i](x + y)
|
46 |
+
x = x * x_mask
|
47 |
+
return x
|
48 |
+
|
49 |
+
|
50 |
+
class Decoder(nn.Module):
|
51 |
+
def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
|
52 |
+
super().__init__()
|
53 |
+
self.hidden_channels = hidden_channels
|
54 |
+
self.filter_channels = filter_channels
|
55 |
+
self.n_heads = n_heads
|
56 |
+
self.n_layers = n_layers
|
57 |
+
self.kernel_size = kernel_size
|
58 |
+
self.p_dropout = p_dropout
|
59 |
+
self.proximal_bias = proximal_bias
|
60 |
+
self.proximal_init = proximal_init
|
61 |
+
|
62 |
+
self.drop = nn.Dropout(p_dropout)
|
63 |
+
self.self_attn_layers = nn.ModuleList()
|
64 |
+
self.norm_layers_0 = nn.ModuleList()
|
65 |
+
self.encdec_attn_layers = nn.ModuleList()
|
66 |
+
self.norm_layers_1 = nn.ModuleList()
|
67 |
+
self.ffn_layers = nn.ModuleList()
|
68 |
+
self.norm_layers_2 = nn.ModuleList()
|
69 |
+
for i in range(self.n_layers):
|
70 |
+
self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
|
71 |
+
self.norm_layers_0.append(LayerNorm(hidden_channels))
|
72 |
+
self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
|
73 |
+
self.norm_layers_1.append(LayerNorm(hidden_channels))
|
74 |
+
self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
|
75 |
+
self.norm_layers_2.append(LayerNorm(hidden_channels))
|
76 |
+
|
77 |
+
def forward(self, x, x_mask, h, h_mask):
|
78 |
+
"""
|
79 |
+
x: decoder input
|
80 |
+
h: encoder output
|
81 |
+
"""
|
82 |
+
self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
|
83 |
+
encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
|
84 |
+
x = x * x_mask
|
85 |
+
for i in range(self.n_layers):
|
86 |
+
y = self.self_attn_layers[i](x, x, self_attn_mask)
|
87 |
+
y = self.drop(y)
|
88 |
+
x = self.norm_layers_0[i](x + y)
|
89 |
+
|
90 |
+
y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
|
91 |
+
y = self.drop(y)
|
92 |
+
x = self.norm_layers_1[i](x + y)
|
93 |
+
|
94 |
+
y = self.ffn_layers[i](x, x_mask)
|
95 |
+
y = self.drop(y)
|
96 |
+
x = self.norm_layers_2[i](x + y)
|
97 |
+
x = x * x_mask
|
98 |
+
return x
|
99 |
+
|
100 |
+
|
101 |
+
class MultiHeadAttention(nn.Module):
|
102 |
+
def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
|
103 |
+
super().__init__()
|
104 |
+
assert channels % n_heads == 0
|
105 |
+
|
106 |
+
self.channels = channels
|
107 |
+
self.out_channels = out_channels
|
108 |
+
self.n_heads = n_heads
|
109 |
+
self.p_dropout = p_dropout
|
110 |
+
self.window_size = window_size
|
111 |
+
self.heads_share = heads_share
|
112 |
+
self.block_length = block_length
|
113 |
+
self.proximal_bias = proximal_bias
|
114 |
+
self.proximal_init = proximal_init
|
115 |
+
self.attn = None
|
116 |
+
|
117 |
+
self.k_channels = channels // n_heads
|
118 |
+
self.conv_q = nn.Conv1d(channels, channels, 1)
|
119 |
+
self.conv_k = nn.Conv1d(channels, channels, 1)
|
120 |
+
self.conv_v = nn.Conv1d(channels, channels, 1)
|
121 |
+
self.conv_o = nn.Conv1d(channels, out_channels, 1)
|
122 |
+
self.drop = nn.Dropout(p_dropout)
|
123 |
+
|
124 |
+
if window_size is not None:
|
125 |
+
n_heads_rel = 1 if heads_share else n_heads
|
126 |
+
rel_stddev = self.k_channels**-0.5
|
127 |
+
self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
|
128 |
+
self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
|
129 |
+
|
130 |
+
nn.init.xavier_uniform_(self.conv_q.weight)
|
131 |
+
nn.init.xavier_uniform_(self.conv_k.weight)
|
132 |
+
nn.init.xavier_uniform_(self.conv_v.weight)
|
133 |
+
if proximal_init:
|
134 |
+
with torch.no_grad():
|
135 |
+
self.conv_k.weight.copy_(self.conv_q.weight)
|
136 |
+
self.conv_k.bias.copy_(self.conv_q.bias)
|
137 |
+
|
138 |
+
def forward(self, x, c, attn_mask=None):
|
139 |
+
q = self.conv_q(x)
|
140 |
+
k = self.conv_k(c)
|
141 |
+
v = self.conv_v(c)
|
142 |
+
|
143 |
+
x, self.attn = self.attention(q, k, v, mask=attn_mask)
|
144 |
+
|
145 |
+
x = self.conv_o(x)
|
146 |
+
return x
|
147 |
+
|
148 |
+
def attention(self, query, key, value, mask=None):
|
149 |
+
# reshape [b, d, t] -> [b, n_h, t, d_k]
|
150 |
+
b, d, t_s, t_t = (*key.size(), query.size(2))
|
151 |
+
query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
|
152 |
+
key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
|
153 |
+
value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
|
154 |
+
|
155 |
+
scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
|
156 |
+
if self.window_size is not None:
|
157 |
+
assert t_s == t_t, "Relative attention is only available for self-attention."
|
158 |
+
key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
|
159 |
+
rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
|
160 |
+
scores_local = self._relative_position_to_absolute_position(rel_logits)
|
161 |
+
scores = scores + scores_local
|
162 |
+
if self.proximal_bias:
|
163 |
+
assert t_s == t_t, "Proximal bias is only available for self-attention."
|
164 |
+
scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
|
165 |
+
if mask is not None:
|
166 |
+
scores = scores.masked_fill(mask == 0, -1e4)
|
167 |
+
if self.block_length is not None:
|
168 |
+
assert t_s == t_t, "Local attention is only available for self-attention."
|
169 |
+
block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
|
170 |
+
scores = scores.masked_fill(block_mask == 0, -1e4)
|
171 |
+
p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
|
172 |
+
p_attn = self.drop(p_attn)
|
173 |
+
output = torch.matmul(p_attn, value)
|
174 |
+
if self.window_size is not None:
|
175 |
+
relative_weights = self._absolute_position_to_relative_position(p_attn)
|
176 |
+
value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
|
177 |
+
output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
|
178 |
+
output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
|
179 |
+
return output, p_attn
|
180 |
+
|
181 |
+
def _matmul_with_relative_values(self, x, y):
|
182 |
+
"""
|
183 |
+
x: [b, h, l, m]
|
184 |
+
y: [h or 1, m, d]
|
185 |
+
ret: [b, h, l, d]
|
186 |
+
"""
|
187 |
+
ret = torch.matmul(x, y.unsqueeze(0))
|
188 |
+
return ret
|
189 |
+
|
190 |
+
def _matmul_with_relative_keys(self, x, y):
|
191 |
+
"""
|
192 |
+
x: [b, h, l, d]
|
193 |
+
y: [h or 1, m, d]
|
194 |
+
ret: [b, h, l, m]
|
195 |
+
"""
|
196 |
+
ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
|
197 |
+
return ret
|
198 |
+
|
199 |
+
def _get_relative_embeddings(self, relative_embeddings, length):
|
200 |
+
max_relative_position = 2 * self.window_size + 1
|
201 |
+
# Pad first before slice to avoid using cond ops.
|
202 |
+
pad_length = max(length - (self.window_size + 1), 0)
|
203 |
+
slice_start_position = max((self.window_size + 1) - length, 0)
|
204 |
+
slice_end_position = slice_start_position + 2 * length - 1
|
205 |
+
if pad_length > 0:
|
206 |
+
padded_relative_embeddings = F.pad(
|
207 |
+
relative_embeddings,
|
208 |
+
commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
|
209 |
+
else:
|
210 |
+
padded_relative_embeddings = relative_embeddings
|
211 |
+
used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
|
212 |
+
return used_relative_embeddings
|
213 |
+
|
214 |
+
def _relative_position_to_absolute_position(self, x):
|
215 |
+
"""
|
216 |
+
x: [b, h, l, 2*l-1]
|
217 |
+
ret: [b, h, l, l]
|
218 |
+
"""
|
219 |
+
batch, heads, length, _ = x.size()
|
220 |
+
# Concat columns of pad to shift from relative to absolute indexing.
|
221 |
+
x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
|
222 |
+
|
223 |
+
# Concat extra elements so to add up to shape (len+1, 2*len-1).
|
224 |
+
x_flat = x.view([batch, heads, length * 2 * length])
|
225 |
+
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
|
226 |
+
|
227 |
+
# Reshape and slice out the padded elements.
|
228 |
+
x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
|
229 |
+
return x_final
|
230 |
+
|
231 |
+
def _absolute_position_to_relative_position(self, x):
|
232 |
+
"""
|
233 |
+
x: [b, h, l, l]
|
234 |
+
ret: [b, h, l, 2*l-1]
|
235 |
+
"""
|
236 |
+
batch, heads, length, _ = x.size()
|
237 |
+
# padd along column
|
238 |
+
x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
|
239 |
+
x_flat = x.view([batch, heads, length**2 + length*(length -1)])
|
240 |
+
# add 0's in the beginning that will skew the elements after reshape
|
241 |
+
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
|
242 |
+
x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
|
243 |
+
return x_final
|
244 |
+
|
245 |
+
def _attention_bias_proximal(self, length):
|
246 |
+
"""Bias for self-attention to encourage attention to close positions.
|
247 |
+
Args:
|
248 |
+
length: an integer scalar.
|
249 |
+
Returns:
|
250 |
+
a Tensor with shape [1, 1, length, length]
|
251 |
+
"""
|
252 |
+
r = torch.arange(length, dtype=torch.float32)
|
253 |
+
diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
|
254 |
+
return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
|
255 |
+
|
256 |
+
|
257 |
+
class FFN(nn.Module):
|
258 |
+
def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
|
259 |
+
super().__init__()
|
260 |
+
self.in_channels = in_channels
|
261 |
+
self.out_channels = out_channels
|
262 |
+
self.filter_channels = filter_channels
|
263 |
+
self.kernel_size = kernel_size
|
264 |
+
self.p_dropout = p_dropout
|
265 |
+
self.activation = activation
|
266 |
+
self.causal = causal
|
267 |
+
|
268 |
+
if causal:
|
269 |
+
self.padding = self._causal_padding
|
270 |
+
else:
|
271 |
+
self.padding = self._same_padding
|
272 |
+
|
273 |
+
self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
|
274 |
+
self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
|
275 |
+
self.drop = nn.Dropout(p_dropout)
|
276 |
+
|
277 |
+
def forward(self, x, x_mask):
|
278 |
+
x = self.conv_1(self.padding(x * x_mask))
|
279 |
+
if self.activation == "gelu":
|
280 |
+
x = x * torch.sigmoid(1.702 * x)
|
281 |
+
else:
|
282 |
+
x = torch.relu(x)
|
283 |
+
x = self.drop(x)
|
284 |
+
x = self.conv_2(self.padding(x * x_mask))
|
285 |
+
return x * x_mask
|
286 |
+
|
287 |
+
def _causal_padding(self, x):
|
288 |
+
if self.kernel_size == 1:
|
289 |
+
return x
|
290 |
+
pad_l = self.kernel_size - 1
|
291 |
+
pad_r = 0
|
292 |
+
padding = [[0, 0], [0, 0], [pad_l, pad_r]]
|
293 |
+
x = F.pad(x, commons.convert_pad_shape(padding))
|
294 |
+
return x
|
295 |
+
|
296 |
+
def _same_padding(self, x):
|
297 |
+
if self.kernel_size == 1:
|
298 |
+
return x
|
299 |
+
pad_l = (self.kernel_size - 1) // 2
|
300 |
+
pad_r = self.kernel_size // 2
|
301 |
+
padding = [[0, 0], [0, 0], [pad_l, pad_r]]
|
302 |
+
x = F.pad(x, commons.convert_pad_shape(padding))
|
303 |
+
return x
|
commons.py
ADDED
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import numpy as np
|
3 |
+
import torch
|
4 |
+
from torch import nn
|
5 |
+
from torch.nn import functional as F
|
6 |
+
|
7 |
+
|
8 |
+
def init_weights(m, mean=0.0, std=0.01):
|
9 |
+
classname = m.__class__.__name__
|
10 |
+
if classname.find("Conv") != -1:
|
11 |
+
m.weight.data.normal_(mean, std)
|
12 |
+
|
13 |
+
|
14 |
+
def get_padding(kernel_size, dilation=1):
|
15 |
+
return int((kernel_size*dilation - dilation)/2)
|
16 |
+
|
17 |
+
|
18 |
+
def convert_pad_shape(pad_shape):
|
19 |
+
l = pad_shape[::-1]
|
20 |
+
pad_shape = [item for sublist in l for item in sublist]
|
21 |
+
return pad_shape
|
22 |
+
|
23 |
+
|
24 |
+
def intersperse(lst, item):
|
25 |
+
result = [item] * (len(lst) * 2 + 1)
|
26 |
+
result[1::2] = lst
|
27 |
+
return result
|
28 |
+
|
29 |
+
|
30 |
+
def kl_divergence(m_p, logs_p, m_q, logs_q):
|
31 |
+
"""KL(P||Q)"""
|
32 |
+
kl = (logs_q - logs_p) - 0.5
|
33 |
+
kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
|
34 |
+
return kl
|
35 |
+
|
36 |
+
|
37 |
+
def rand_gumbel(shape):
|
38 |
+
"""Sample from the Gumbel distribution, protect from overflows."""
|
39 |
+
uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
|
40 |
+
return -torch.log(-torch.log(uniform_samples))
|
41 |
+
|
42 |
+
|
43 |
+
def rand_gumbel_like(x):
|
44 |
+
g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
|
45 |
+
return g
|
46 |
+
|
47 |
+
|
48 |
+
def slice_segments(x, ids_str, segment_size=4):
|
49 |
+
ret = torch.zeros_like(x[:, :, :segment_size])
|
50 |
+
for i in range(x.size(0)):
|
51 |
+
idx_str = ids_str[i]
|
52 |
+
idx_end = idx_str + segment_size
|
53 |
+
ret[i] = x[i, :, idx_str:idx_end]
|
54 |
+
return ret
|
55 |
+
|
56 |
+
|
57 |
+
def rand_slice_segments(x, x_lengths=None, segment_size=4):
|
58 |
+
b, d, t = x.size()
|
59 |
+
if x_lengths is None:
|
60 |
+
x_lengths = t
|
61 |
+
ids_str_max = x_lengths - segment_size + 1
|
62 |
+
ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
|
63 |
+
ret = slice_segments(x, ids_str, segment_size)
|
64 |
+
return ret, ids_str
|
65 |
+
|
66 |
+
|
67 |
+
def get_timing_signal_1d(
|
68 |
+
length, channels, min_timescale=1.0, max_timescale=1.0e4):
|
69 |
+
position = torch.arange(length, dtype=torch.float)
|
70 |
+
num_timescales = channels // 2
|
71 |
+
log_timescale_increment = (
|
72 |
+
math.log(float(max_timescale) / float(min_timescale)) /
|
73 |
+
(num_timescales - 1))
|
74 |
+
inv_timescales = min_timescale * torch.exp(
|
75 |
+
torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
|
76 |
+
scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
|
77 |
+
signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
|
78 |
+
signal = F.pad(signal, [0, 0, 0, channels % 2])
|
79 |
+
signal = signal.view(1, channels, length)
|
80 |
+
return signal
|
81 |
+
|
82 |
+
|
83 |
+
def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
|
84 |
+
b, channels, length = x.size()
|
85 |
+
signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
|
86 |
+
return x + signal.to(dtype=x.dtype, device=x.device)
|
87 |
+
|
88 |
+
|
89 |
+
def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
|
90 |
+
b, channels, length = x.size()
|
91 |
+
signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
|
92 |
+
return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
|
93 |
+
|
94 |
+
|
95 |
+
def subsequent_mask(length):
|
96 |
+
mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
|
97 |
+
return mask
|
98 |
+
|
99 |
+
|
100 |
+
@torch.jit.script
|
101 |
+
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
|
102 |
+
n_channels_int = n_channels[0]
|
103 |
+
in_act = input_a + input_b
|
104 |
+
t_act = torch.tanh(in_act[:, :n_channels_int, :])
|
105 |
+
s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
|
106 |
+
acts = t_act * s_act
|
107 |
+
return acts
|
108 |
+
|
109 |
+
|
110 |
+
def convert_pad_shape(pad_shape):
|
111 |
+
l = pad_shape[::-1]
|
112 |
+
pad_shape = [item for sublist in l for item in sublist]
|
113 |
+
return pad_shape
|
114 |
+
|
115 |
+
|
116 |
+
def shift_1d(x):
|
117 |
+
x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
|
118 |
+
return x
|
119 |
+
|
120 |
+
|
121 |
+
def sequence_mask(length, max_length=None):
|
122 |
+
if max_length is None:
|
123 |
+
max_length = length.max()
|
124 |
+
x = torch.arange(max_length, dtype=length.dtype, device=length.device)
|
125 |
+
return x.unsqueeze(0) < length.unsqueeze(1)
|
126 |
+
|
127 |
+
|
128 |
+
def generate_path(duration, mask):
|
129 |
+
"""
|
130 |
+
duration: [b, 1, t_x]
|
131 |
+
mask: [b, 1, t_y, t_x]
|
132 |
+
"""
|
133 |
+
device = duration.device
|
134 |
+
|
135 |
+
b, _, t_y, t_x = mask.shape
|
136 |
+
cum_duration = torch.cumsum(duration, -1)
|
137 |
+
|
138 |
+
cum_duration_flat = cum_duration.view(b * t_x)
|
139 |
+
path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
|
140 |
+
path = path.view(b, t_x, t_y)
|
141 |
+
path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
|
142 |
+
path = path.unsqueeze(1).transpose(2,3) * mask
|
143 |
+
return path
|
144 |
+
|
145 |
+
|
146 |
+
def clip_grad_value_(parameters, clip_value, norm_type=2):
|
147 |
+
if isinstance(parameters, torch.Tensor):
|
148 |
+
parameters = [parameters]
|
149 |
+
parameters = list(filter(lambda p: p.grad is not None, parameters))
|
150 |
+
norm_type = float(norm_type)
|
151 |
+
if clip_value is not None:
|
152 |
+
clip_value = float(clip_value)
|
153 |
+
|
154 |
+
total_norm = 0
|
155 |
+
for p in parameters:
|
156 |
+
param_norm = p.grad.data.norm(norm_type)
|
157 |
+
total_norm += param_norm.item() ** norm_type
|
158 |
+
if clip_value is not None:
|
159 |
+
p.grad.data.clamp_(min=-clip_value, max=clip_value)
|
160 |
+
total_norm = total_norm ** (1. / norm_type)
|
161 |
+
return total_norm
|
configs/vtubers.json
ADDED
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"train": {
|
3 |
+
"log_interval": 200,
|
4 |
+
"eval_interval": 400,
|
5 |
+
"seed": 1234,
|
6 |
+
"epochs": 10000,
|
7 |
+
"learning_rate": 2e-4,
|
8 |
+
"betas": [0.8, 0.99],
|
9 |
+
"eps": 1e-9,
|
10 |
+
"batch_size": 8,
|
11 |
+
"fp16_run": false,
|
12 |
+
"lr_decay": 0.999875,
|
13 |
+
"segment_size": 8192,
|
14 |
+
"init_lr_ratio": 1,
|
15 |
+
"warmup_epochs": 0,
|
16 |
+
"c_mel": 45,
|
17 |
+
"c_kl": 1.0
|
18 |
+
},
|
19 |
+
"data": {
|
20 |
+
"training_files":"filelists/train.txt.cleaned",
|
21 |
+
"validation_files":"filelists/val.txt.cleaned",
|
22 |
+
"text_cleaners":["japanese_cleaners"],
|
23 |
+
"max_wav_value": 32768.0,
|
24 |
+
"sampling_rate": 22050,
|
25 |
+
"filter_length": 1024,
|
26 |
+
"hop_length": 256,
|
27 |
+
"win_length": 1024,
|
28 |
+
"n_mel_channels": 80,
|
29 |
+
"mel_fmin": 0.0,
|
30 |
+
"mel_fmax": null,
|
31 |
+
"add_blank": true,
|
32 |
+
"n_speakers": 7,
|
33 |
+
"cleaned_text": true
|
34 |
+
},
|
35 |
+
"model": {
|
36 |
+
"inter_channels": 192,
|
37 |
+
"hidden_channels": 192,
|
38 |
+
"filter_channels": 768,
|
39 |
+
"n_heads": 2,
|
40 |
+
"n_layers": 6,
|
41 |
+
"kernel_size": 3,
|
42 |
+
"p_dropout": 0.1,
|
43 |
+
"resblock": "1",
|
44 |
+
"resblock_kernel_sizes": [3,7,11],
|
45 |
+
"resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
|
46 |
+
"upsample_rates": [8,8,2,2],
|
47 |
+
"upsample_initial_channel": 512,
|
48 |
+
"upsample_kernel_sizes": [16,16,4,4],
|
49 |
+
"n_layers_q": 3,
|
50 |
+
"use_spectral_norm": false,
|
51 |
+
"gin_channels": 256
|
52 |
+
}
|
53 |
+
}
|
data_utils.py
ADDED
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import time
|
2 |
+
import os
|
3 |
+
import random
|
4 |
+
import numpy as np
|
5 |
+
import torch
|
6 |
+
import torch.utils.data
|
7 |
+
|
8 |
+
import commons
|
9 |
+
from mel_processing import spectrogram_torch
|
10 |
+
from utils import load_wav_to_torch, load_filepaths_and_text
|
11 |
+
from text import text_to_sequence, cleaned_text_to_sequence
|
12 |
+
|
13 |
+
"""Multi speaker version"""
|
14 |
+
class TextAudioSpeakerLoader(torch.utils.data.Dataset):
|
15 |
+
"""
|
16 |
+
1) loads audio, speaker_id, text pairs
|
17 |
+
2) normalizes text and converts them to sequences of integers
|
18 |
+
3) computes spectrograms from audio files.
|
19 |
+
"""
|
20 |
+
def __init__(self, audiopaths_sid_text, hparams):
|
21 |
+
self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
|
22 |
+
self.text_cleaners = hparams.text_cleaners
|
23 |
+
self.max_wav_value = hparams.max_wav_value
|
24 |
+
self.sampling_rate = hparams.sampling_rate
|
25 |
+
self.filter_length = hparams.filter_length
|
26 |
+
self.hop_length = hparams.hop_length
|
27 |
+
self.win_length = hparams.win_length
|
28 |
+
self.sampling_rate = hparams.sampling_rate
|
29 |
+
|
30 |
+
self.cleaned_text = getattr(hparams, "cleaned_text", False)
|
31 |
+
|
32 |
+
self.add_blank = hparams.add_blank
|
33 |
+
self.min_text_len = getattr(hparams, "min_text_len", 1)
|
34 |
+
self.max_text_len = getattr(hparams, "max_text_len", 190)
|
35 |
+
|
36 |
+
random.seed(1234)
|
37 |
+
random.shuffle(self.audiopaths_sid_text)
|
38 |
+
self._filter()
|
39 |
+
|
40 |
+
def _filter(self):
|
41 |
+
"""
|
42 |
+
Filter text & store spec lengths
|
43 |
+
"""
|
44 |
+
# Store spectrogram lengths for Bucketing
|
45 |
+
# wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
|
46 |
+
# spec_length = wav_length // hop_length
|
47 |
+
|
48 |
+
audiopaths_sid_text_new = []
|
49 |
+
lengths = []
|
50 |
+
for audiopath, sid, text in self.audiopaths_sid_text:
|
51 |
+
if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
|
52 |
+
audiopaths_sid_text_new.append([audiopath, sid, text])
|
53 |
+
lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
|
54 |
+
self.audiopaths_sid_text = audiopaths_sid_text_new
|
55 |
+
self.lengths = lengths
|
56 |
+
|
57 |
+
def get_audio_text_speaker_pair(self, audiopath_sid_text):
|
58 |
+
# separate filename, speaker_id and text
|
59 |
+
audiopath, sid, text = audiopath_sid_text[0], audiopath_sid_text[1], audiopath_sid_text[2]
|
60 |
+
text = self.get_text(text)
|
61 |
+
spec, wav = self.get_audio(audiopath)
|
62 |
+
sid = self.get_sid(sid)
|
63 |
+
emo = torch.FloatTensor(np.load(audiopath+".emo.npy"))
|
64 |
+
return (text, spec, wav, sid, emo)
|
65 |
+
|
66 |
+
def get_audio(self, filename):
|
67 |
+
audio, sampling_rate = load_wav_to_torch(filename)
|
68 |
+
if sampling_rate != self.sampling_rate:
|
69 |
+
raise ValueError("{} {} SR doesn't match target {} SR".format(
|
70 |
+
sampling_rate, self.sampling_rate))
|
71 |
+
audio_norm = audio / self.max_wav_value
|
72 |
+
audio_norm = audio_norm.unsqueeze(0)
|
73 |
+
spec_filename = filename.replace(".wav", ".spec.pt")
|
74 |
+
if os.path.exists(spec_filename):
|
75 |
+
spec = torch.load(spec_filename)
|
76 |
+
else:
|
77 |
+
spec = spectrogram_torch(audio_norm, self.filter_length,
|
78 |
+
self.sampling_rate, self.hop_length, self.win_length,
|
79 |
+
center=False)
|
80 |
+
spec = torch.squeeze(spec, 0)
|
81 |
+
torch.save(spec, spec_filename)
|
82 |
+
return spec, audio_norm
|
83 |
+
|
84 |
+
def get_text(self, text):
|
85 |
+
if self.cleaned_text:
|
86 |
+
text_norm = cleaned_text_to_sequence(text)
|
87 |
+
else:
|
88 |
+
text_norm = text_to_sequence(text, self.text_cleaners)
|
89 |
+
if self.add_blank:
|
90 |
+
text_norm = commons.intersperse(text_norm, 0)
|
91 |
+
text_norm = torch.LongTensor(text_norm)
|
92 |
+
return text_norm
|
93 |
+
|
94 |
+
def get_sid(self, sid):
|
95 |
+
sid = torch.LongTensor([int(sid)])
|
96 |
+
return sid
|
97 |
+
|
98 |
+
def __getitem__(self, index):
|
99 |
+
return self.get_audio_text_speaker_pair(self.audiopaths_sid_text[index])
|
100 |
+
|
101 |
+
def __len__(self):
|
102 |
+
return len(self.audiopaths_sid_text)
|
103 |
+
|
104 |
+
|
105 |
+
class TextAudioSpeakerCollate():
|
106 |
+
""" Zero-pads model inputs and targets
|
107 |
+
"""
|
108 |
+
def __init__(self, return_ids=False):
|
109 |
+
self.return_ids = return_ids
|
110 |
+
|
111 |
+
def __call__(self, batch):
|
112 |
+
"""Collate's training batch from normalized text, audio and speaker identities
|
113 |
+
PARAMS
|
114 |
+
------
|
115 |
+
batch: [text_normalized, spec_normalized, wav_normalized, sid]
|
116 |
+
"""
|
117 |
+
# Right zero-pad all one-hot text sequences to max input length
|
118 |
+
_, ids_sorted_decreasing = torch.sort(
|
119 |
+
torch.LongTensor([x[1].size(1) for x in batch]),
|
120 |
+
dim=0, descending=True)
|
121 |
+
|
122 |
+
max_text_len = max([len(x[0]) for x in batch])
|
123 |
+
max_spec_len = max([x[1].size(1) for x in batch])
|
124 |
+
max_wav_len = max([x[2].size(1) for x in batch])
|
125 |
+
|
126 |
+
text_lengths = torch.LongTensor(len(batch))
|
127 |
+
spec_lengths = torch.LongTensor(len(batch))
|
128 |
+
wav_lengths = torch.LongTensor(len(batch))
|
129 |
+
sid = torch.LongTensor(len(batch))
|
130 |
+
|
131 |
+
text_padded = torch.LongTensor(len(batch), max_text_len)
|
132 |
+
spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
|
133 |
+
wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
|
134 |
+
emo = torch.FloatTensor(len(batch), 1024)
|
135 |
+
|
136 |
+
text_padded.zero_()
|
137 |
+
spec_padded.zero_()
|
138 |
+
wav_padded.zero_()
|
139 |
+
emo.zero_()
|
140 |
+
|
141 |
+
for i in range(len(ids_sorted_decreasing)):
|
142 |
+
row = batch[ids_sorted_decreasing[i]]
|
143 |
+
|
144 |
+
text = row[0]
|
145 |
+
text_padded[i, :text.size(0)] = text
|
146 |
+
text_lengths[i] = text.size(0)
|
147 |
+
|
148 |
+
spec = row[1]
|
149 |
+
spec_padded[i, :, :spec.size(1)] = spec
|
150 |
+
spec_lengths[i] = spec.size(1)
|
151 |
+
|
152 |
+
wav = row[2]
|
153 |
+
wav_padded[i, :, :wav.size(1)] = wav
|
154 |
+
wav_lengths[i] = wav.size(1)
|
155 |
+
|
156 |
+
sid[i] = row[3]
|
157 |
+
|
158 |
+
emo[i, :] = row[4]
|
159 |
+
|
160 |
+
if self.return_ids:
|
161 |
+
return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, ids_sorted_decreasing
|
162 |
+
return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid,emo
|
163 |
+
|
164 |
+
|
165 |
+
class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
|
166 |
+
"""
|
167 |
+
Maintain similar input lengths in a batch.
|
168 |
+
Length groups are specified by boundaries.
|
169 |
+
Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
|
170 |
+
|
171 |
+
It removes samples which are not included in the boundaries.
|
172 |
+
Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
|
173 |
+
"""
|
174 |
+
def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True):
|
175 |
+
super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
|
176 |
+
self.lengths = dataset.lengths
|
177 |
+
self.batch_size = batch_size
|
178 |
+
self.boundaries = boundaries
|
179 |
+
|
180 |
+
self.buckets, self.num_samples_per_bucket = self._create_buckets()
|
181 |
+
self.total_size = sum(self.num_samples_per_bucket)
|
182 |
+
self.num_samples = self.total_size // self.num_replicas
|
183 |
+
|
184 |
+
def _create_buckets(self):
|
185 |
+
buckets = [[] for _ in range(len(self.boundaries) - 1)]
|
186 |
+
for i in range(len(self.lengths)):
|
187 |
+
length = self.lengths[i]
|
188 |
+
idx_bucket = self._bisect(length)
|
189 |
+
if idx_bucket != -1:
|
190 |
+
buckets[idx_bucket].append(i)
|
191 |
+
|
192 |
+
for i in range(len(buckets) - 1, 0, -1):
|
193 |
+
if len(buckets[i]) == 0:
|
194 |
+
buckets.pop(i)
|
195 |
+
self.boundaries.pop(i+1)
|
196 |
+
|
197 |
+
num_samples_per_bucket = []
|
198 |
+
for i in range(len(buckets)):
|
199 |
+
len_bucket = len(buckets[i])
|
200 |
+
total_batch_size = self.num_replicas * self.batch_size
|
201 |
+
rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size
|
202 |
+
num_samples_per_bucket.append(len_bucket + rem)
|
203 |
+
return buckets, num_samples_per_bucket
|
204 |
+
|
205 |
+
def __iter__(self):
|
206 |
+
# deterministically shuffle based on epoch
|
207 |
+
g = torch.Generator()
|
208 |
+
g.manual_seed(self.epoch)
|
209 |
+
|
210 |
+
indices = []
|
211 |
+
if self.shuffle:
|
212 |
+
for bucket in self.buckets:
|
213 |
+
indices.append(torch.randperm(len(bucket), generator=g).tolist())
|
214 |
+
else:
|
215 |
+
for bucket in self.buckets:
|
216 |
+
indices.append(list(range(len(bucket))))
|
217 |
+
|
218 |
+
batches = []
|
219 |
+
for i in range(len(self.buckets)):
|
220 |
+
bucket = self.buckets[i]
|
221 |
+
len_bucket = len(bucket)
|
222 |
+
ids_bucket = indices[i]
|
223 |
+
num_samples_bucket = self.num_samples_per_bucket[i]
|
224 |
+
|
225 |
+
# add extra samples to make it evenly divisible
|
226 |
+
rem = num_samples_bucket - len_bucket
|
227 |
+
ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
|
228 |
+
|
229 |
+
# subsample
|
230 |
+
ids_bucket = ids_bucket[self.rank::self.num_replicas]
|
231 |
+
|
232 |
+
# batching
|
233 |
+
for j in range(len(ids_bucket) // self.batch_size):
|
234 |
+
batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]]
|
235 |
+
batches.append(batch)
|
236 |
+
|
237 |
+
if self.shuffle:
|
238 |
+
batch_ids = torch.randperm(len(batches), generator=g).tolist()
|
239 |
+
batches = [batches[i] for i in batch_ids]
|
240 |
+
self.batches = batches
|
241 |
+
|
242 |
+
assert len(self.batches) * self.batch_size == self.num_samples
|
243 |
+
return iter(self.batches)
|
244 |
+
|
245 |
+
def _bisect(self, x, lo=0, hi=None):
|
246 |
+
if hi is None:
|
247 |
+
hi = len(self.boundaries) - 1
|
248 |
+
|
249 |
+
if hi > lo:
|
250 |
+
mid = (hi + lo) // 2
|
251 |
+
if self.boundaries[mid] < x and x <= self.boundaries[mid+1]:
|
252 |
+
return mid
|
253 |
+
elif x <= self.boundaries[mid]:
|
254 |
+
return self._bisect(x, lo, mid)
|
255 |
+
else:
|
256 |
+
return self._bisect(x, mid + 1, hi)
|
257 |
+
else:
|
258 |
+
return -1
|
259 |
+
|
260 |
+
def __len__(self):
|
261 |
+
return self.num_samples // self.batch_size
|
emotion_extract.py
ADDED
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
import torch.nn as nn
|
3 |
+
from transformers import Wav2Vec2Processor
|
4 |
+
from transformers.models.wav2vec2.modeling_wav2vec2 import (
|
5 |
+
Wav2Vec2Model,
|
6 |
+
Wav2Vec2PreTrainedModel,
|
7 |
+
)
|
8 |
+
import os
|
9 |
+
import librosa
|
10 |
+
import numpy as np
|
11 |
+
|
12 |
+
|
13 |
+
class RegressionHead(nn.Module):
|
14 |
+
r"""Classification head."""
|
15 |
+
|
16 |
+
def __init__(self, config):
|
17 |
+
super().__init__()
|
18 |
+
|
19 |
+
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
20 |
+
self.dropout = nn.Dropout(config.final_dropout)
|
21 |
+
self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
|
22 |
+
|
23 |
+
def forward(self, features, **kwargs):
|
24 |
+
x = features
|
25 |
+
x = self.dropout(x)
|
26 |
+
x = self.dense(x)
|
27 |
+
x = torch.tanh(x)
|
28 |
+
x = self.dropout(x)
|
29 |
+
x = self.out_proj(x)
|
30 |
+
|
31 |
+
return x
|
32 |
+
|
33 |
+
|
34 |
+
class EmotionModel(Wav2Vec2PreTrainedModel):
|
35 |
+
r"""Speech emotion classifier."""
|
36 |
+
|
37 |
+
def __init__(self, config):
|
38 |
+
super().__init__(config)
|
39 |
+
|
40 |
+
self.config = config
|
41 |
+
self.wav2vec2 = Wav2Vec2Model(config)
|
42 |
+
self.classifier = RegressionHead(config)
|
43 |
+
self.init_weights()
|
44 |
+
|
45 |
+
def forward(
|
46 |
+
self,
|
47 |
+
input_values,
|
48 |
+
):
|
49 |
+
outputs = self.wav2vec2(input_values)
|
50 |
+
hidden_states = outputs[0]
|
51 |
+
hidden_states = torch.mean(hidden_states, dim=1)
|
52 |
+
logits = self.classifier(hidden_states)
|
53 |
+
|
54 |
+
return hidden_states, logits
|
55 |
+
|
56 |
+
|
57 |
+
# load model from hub
|
58 |
+
device = 'cuda' if torch.cuda.is_available() else "cpu"
|
59 |
+
model_name = 'audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'
|
60 |
+
processor = Wav2Vec2Processor.from_pretrained(model_name)
|
61 |
+
model = EmotionModel.from_pretrained(model_name).to(device)
|
62 |
+
|
63 |
+
|
64 |
+
def process_func(
|
65 |
+
x: np.ndarray,
|
66 |
+
sampling_rate: int,
|
67 |
+
embeddings: bool = False,
|
68 |
+
) -> np.ndarray:
|
69 |
+
r"""Predict emotions or extract embeddings from raw audio signal."""
|
70 |
+
|
71 |
+
# run through processor to normalize signal
|
72 |
+
# always returns a batch, so we just get the first entry
|
73 |
+
# then we put it on the device
|
74 |
+
y = processor(x, sampling_rate=sampling_rate)
|
75 |
+
y = y['input_values'][0]
|
76 |
+
y = torch.from_numpy(y).to(device)
|
77 |
+
|
78 |
+
# run through model
|
79 |
+
with torch.no_grad():
|
80 |
+
y = model(y)[0 if embeddings else 1]
|
81 |
+
|
82 |
+
# convert to numpy
|
83 |
+
y = y.detach().cpu().numpy()
|
84 |
+
|
85 |
+
return y
|
86 |
+
#
|
87 |
+
#
|
88 |
+
# def disp(rootpath, wavname):
|
89 |
+
# wav, sr = librosa.load(f"{rootpath}/{wavname}", 16000)
|
90 |
+
# display(ipd.Audio(wav, rate=sr))
|
91 |
+
|
92 |
+
rootpath = "dataset/nene"
|
93 |
+
embs = []
|
94 |
+
wavnames = []
|
95 |
+
def extract_dir(path):
|
96 |
+
rootpath = path
|
97 |
+
for idx, wavname in enumerate(os.listdir(rootpath)):
|
98 |
+
wav, sr =librosa.load(f"{rootpath}/{wavname}", 16000)
|
99 |
+
emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
|
100 |
+
embs.append(emb)
|
101 |
+
wavnames.append(wavname)
|
102 |
+
np.save(f"{rootpath}/{wavname}.emo.npy", emb.squeeze(0))
|
103 |
+
print(idx, wavname)
|
104 |
+
|
105 |
+
def extract_wav(path):
|
106 |
+
wav, sr = librosa.load(path, 16000)
|
107 |
+
emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
|
108 |
+
return emb
|
109 |
+
|
110 |
+
if __name__ == '__main__':
|
111 |
+
for spk in ["serena", "koni", "nyaru","shanoa", "mana"]:
|
112 |
+
extract_dir(f"dataset/{spk}")
|
filelists/train.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
filelists/train.txt.cleaned
ADDED
The diff for this file is too large to render.
See raw diff
|
|
filelists/val.txt
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
dataset/nene/nen001_001.wav|0|はい?呼びました?
|
2 |
+
dataset/nene/nen001_002.wav|0|驚かせたならごめんなさい。通りかかったときにちょうど名前が聞こえてきたので
|
3 |
+
dataset/nene/nen001_003.wav|0|私に何か用ですか?
|
4 |
+
dataset/nene/nen001_004.wav|0|いえ、少し気になっただけですから、別に怒っているわけじゃないです。気にしないで下さい
|
5 |
+
dataset/nene/nen001_005.wav|0|それより仮屋さん、例の件ですが――
|
6 |
+
dataset/nene/nen001_006.wav|0|喜んでもらえたならなによりです
|
7 |
+
dataset/nene/nen001_007.wav|0|また何かあったら、いつでも部室に来て下さい
|
8 |
+
dataset/nene/nen001_008.wav|0|大したことじゃないので。それより体調の方はもういいんですか?
|
9 |
+
dataset/nene/nen001_009.wav|0|それはよかったです。他に困ったことはありますか?
|
10 |
+
dataset/nene/nen001_010.wav|0|何かあったらいつでも話して下さい。学院のことじゃなく、私事に関することでも何でも
|
11 |
+
dataset/nene/nen001_011.wav|0|はい、いつでもどうぞ
|
12 |
+
dataset/nene/nen001_012.wav|0|保科君も
|
13 |
+
dataset/nene/nen001_013.wav|0|もし何か困ったことがあれば、力になりますから
|
14 |
+
dataset/nene/nen001_014.wav|0|そうですか?私には、なにか悩み事があるように見えたりしましたけど……
|
15 |
+
dataset/nene/nen001_015.wav|0|――なんて、言ってみただけですから、深い意味はありませんよ
|
16 |
+
dataset/nene/nen001_016.wav|0|いえ、そういうわけじゃなくって……ただ何となく。そんな気がしただけですから
|
17 |
+
dataset/nene/nen001_017.wav|0|あ、いえ、絶対に秘密というわけじゃないので気にしないで下さい
|
18 |
+
dataset/nene/nen001_018.wav|0|好奇心だけで来られると困るので、本当に悩んでる人以外には、広めないようにお願いしているだけです
|
19 |
+
dataset/nene/nen001_019.wav|0|そんな仰々しい話じゃなく……私は、オカ研に所属しているんです
|
20 |
+
dataset/nene/nen001_020.wav|0|はい。そのオカ研です
|
21 |
+
dataset/nene/nen001_021.wav|0|そうですよ。実は私たちが入学する前からあった部活なんです。今は部員がいなくて、所属してるのは私一人ですが
|
22 |
+
dataset/nene/nen001_022.wav|0|なので最近は、学生会の方からは結構つつかれてて……活動も、発表などを意欲的にしているわけじゃありませんからね
|
23 |
+
dataset/nene/nen001_023.wav|0|いえ、私は占いを。オカルトと言っても幅は広いので。それに部には私しかいませんから、結構好きにできるんです
|
24 |
+
dataset/nene/nen001_024.wav|0|さすがに白蛇占いはできませんよ
|
25 |
+
dataset/nene/nen001_025.wav|0|一応。知ってはいますけど、私にできるのはタロットぐらいです。あくまで趣味程度ですから
|
26 |
+
dataset/nene/nen001_026.wav|0|あくまで占いの延長線上のものですから。人生相談なんていうほど大層な物じゃありません
|
27 |
+
dataset/nene/nen001_027.wav|0|でも……もし保科君も嫌いでなければ、いつでも部室に来て下さい
|
28 |
+
dataset/nene/nen001_028.wav|0|あ、はい。すぐに行きます
|
29 |
+
dataset/nene/nen001_029.wav|0|それじゃあ私はこれで
|
30 |
+
dataset/nene/nen001_030.wav|0|あの、先生
|
31 |
+
dataset/nene/nen001_031.wav|0|はい。気付いたらこんな時間になってしまって
|
32 |
+
dataset/nene/nen001_032.wav|0|図書室は……もう誰もいないんですか?
|
33 |
+
dataset/nene/nen001_033.wav|0|その前にちょっと調べ物をさせて欲しいんですが、鍵を貸してもらえませんか?
|
34 |
+
dataset/nene/nen001_034.wav|0|タロットカードのことで少々。時間はかかりません。5分……は無理かも……でも20分もあれば終わりますから……お願いします
|
35 |
+
dataset/nene/nen001_035.wav|0|わかりました。ありがとう……ございます
|
36 |
+
dataset/nene/nen001_036.wav|0|は、はい………気をつけます……ハァ、ハァ……
|
37 |
+
dataset/nene/nen001_037.wav|0|……ハァ……ハァ……
|
38 |
+
dataset/nene/nen001_038.wav|0|……大丈夫、ですよね……んっ……
|
39 |
+
dataset/nene/nen001_039.wav|0|……ハァ……ハァ……
|
40 |
+
dataset/nene/nen001_040.wav|0|んっ、んん……
|
41 |
+
dataset/nene/nen001_041.wav|0|あ……あっ、んっ……んんン、んッ、んッ、んぅぅッ……
|
42 |
+
dataset/nene/nen001_042.wav|0|んっ、ふぅぅ……はぁ、はぁ、あっ、ああぁぁ……ふぁぁぁ……
|
43 |
+
dataset/nene/nen001_043.wav|0|はぁ、はぁぁ……ん、ん、んっ……はぁぁ……ぁ、ぁ、ぁ……ぁぁ……ンッ……!
|
44 |
+
dataset/nene/nen001_044.wav|0|あぁ、もう……こ、んなの、最低ですッ……はぁ、はぁ……学院内で、おっ、オナニー……をするなんてっ、ん、んんっ
|
45 |
+
dataset/nene/nen001_045.wav|0|あっ、ああぁぁ……はぁ、はぁ、はぁぁぁ……ん、んンッッ
|
46 |
+
dataset/nene/nen001_046.wav|0|はぁ、はぁ……私、��んてこと……あっ、あぁぁ……でも、気持ちよくて止まりません……ん、んんッ、ふぁ、あ、あぁぁぁ……
|
47 |
+
dataset/nene/nen001_047.wav|0|うっ……あ、あ、あっ、あっ、ひあっ、ん、んはっ……はっ、はっ、んんぅぅッ
|
48 |
+
dataset/nene/nen001_048.wav|0|はぁ……はぁ……んっ、んんん……んぁ、んぁ、あっ……んっ、ぃぃぃッ
|
49 |
+
dataset/nene/nen001_049.wav|0|んっ、んっ、んくっ……ひっ、あっ、ぁっ、ぁっ、んんーーッ……
|
50 |
+
dataset/nene/nen001_050.wav|0|はぁぁぁ……はぁ、はぁ……んんっ、んっ、んんん、んぁ……んぁ、ぁ、ぁ、ぁぁぁぁ……ッ!
|
51 |
+
dataset/nene/nen001_051.wav|0|だめ、早く、しないと……先生が、戻って、きます……こんなところ、見られたら……んっ、んっ、んぁ、んぁ、あ、あ、あ、あ、あ……
|
52 |
+
dataset/nene/nen001_052.wav|0|あ、あ、ん、んんんッ……はっ、はぁ、はぁ、あぁぁもう、本当に、最低ですっ……
|
53 |
+
dataset/nene/nen001_053.wav|0|んッ……んはぁッ、はぁ、はぁ……ンン、んぁ、んぁぁ……あ、あ、あぁぁ……
|
54 |
+
dataset/nene/nen001_054.wav|0|はぁー、はぁーーぁぁ、気持ちいい……あっ、あぃ、あぃっ、いい……本当に……んん
|
55 |
+
dataset/nene/nen001_055.wav|0|早く……こんなこと、早く……先生が、また様子を……見に、きたりしない、内に……はぁ、はぁ、はぁ、はぁ
|
56 |
+
dataset/nene/nen001_056.wav|0|はぁ、はぁ、はぁっ、はぁぁぁ……ぁ、ぁ、ぁ……ぁぁ……ぁぁんッ、んっ、んーッ
|
57 |
+
dataset/nene/nen001_057.wav|0|んんッ、ん、んはぁぁぁーー……はぁ、はぁ、はぁァン、あッ、あん、あぁぁンッ
|
58 |
+
dataset/nene/nen001_058.wav|0|はぁ、はぁ、はぁぁ……ヨダレ、でちゃう……じゅる……んぁっ、はぁ、はぁぁ……はっ、はっ、はっ……じゅる
|
59 |
+
dataset/nene/nen001_059.wav|0|こんな、に、気持ちいいなんて、最低です……本当、図書室でオナニーなんて、最低すぎます……でも、でも
|
60 |
+
dataset/nene/nen001_060.wav|0|あああぁぁ……今は、止められなくて……じゅる……はぁ、はぁぁ……あぁぁぁあぁ……
|
61 |
+
dataset/nene/nen001_061.wav|0|はぁ、はぁ、はぁ……早く……あっ、あっ、あぁぁ……ぁぁぁ、早く、早く
|
62 |
+
dataset/nene/nen001_062.wav|0|早く、イきたい、イきたい……このままイきたいぃ……はぁ、はぁ、はぁ、ぁぁ、ぁっ、ぁっ、あっ、あっ、あぁッ!
|
63 |
+
dataset/nene/nen001_063.wav|0|あっ、あっ、ああぁぁ……痺れて、きたぁ……だめ、イきそう……んっ、だめじゃなくて、あっ、あっ、早く、このまま……早く早く早く
|
filelists/val.txt.cleaned
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
dataset/nene/nen001_001.wav|0|ha↓i? yo↑bima↓ʃIta?
|
2 |
+
dataset/nene/nen001_002.wav|0|o↑doro↓kasetanara go↑meNnasa↓i. to↑orikaka↓Qta to↓kini ʧo↑odo na↑maega kI↑koete ki↓tanode.
|
3 |
+
dataset/nene/nen001_003.wav|0|wa↑taʃini na↑nika↓yoodesUka?
|
4 |
+
dataset/nene/nen001_004.wav|0|i↓e, sU↑ko↓ʃI ki↑ni na↓QtadakedesUkara, be↑tsuni o↑ko↓Qte i↑ru wa↓kejanaidesU. ki↑ni ʃi↑na↓ide ku↑dasa↓i.
|
5 |
+
dataset/nene/nen001_005.wav|0|so↑reyo↓ri ka↑riyasaN, re↓eno ke↓NdesUga----
|
6 |
+
dataset/nene/nen001_006.wav|0|yo↑roko↓Nde mo↑raeta↓nara na↓niyoridesU.
|
7 |
+
dataset/nene/nen001_007.wav|0|ma↑ta na↓nika a↓Qtara, i↓tsudemo bu↑ʃItsuni ki↓te ku↑dasa↓i.
|
8 |
+
dataset/nene/nen001_008.wav|0|ta↓iʃIta ko↑to↓janainode. so↑reyo↓ri ta↑iʧoono ho↓owa mo↓o i↓i N↓desUka?
|
9 |
+
dataset/nene/nen001_009.wav|0|so↑rewa yo↓kaQtadesU. ta↓ni ko↑ma↓Qta ko↑to↓wa a↑rima↓sUka?
|
10 |
+
dataset/nene/nen001_010.wav|0|na↓nika a↓Qtara i↓tsudemo ha↑na↓ʃIte ku↑dasa↓i. ga↑kuiNno ko↑to↓janaku, ʃi↓jini ka↑Nsu↓ru ko↑to↓demo na↓nidemo.
|
11 |
+
dataset/nene/nen001_011.wav|0|ha↓i, i↓tsudemo do↓ozo.
|
12 |
+
dataset/nene/nen001_012.wav|0|ho↓ʃinakuNmo.
|
13 |
+
dataset/nene/nen001_013.wav|0|mo↓ʃi na↓nika ko↑ma↓Qta ko↑to↓ga a↑re↓ba, ʧI↑kara↓ni na↑rima↓sUkara.
|
14 |
+
dataset/nene/nen001_014.wav|0|so↑odesU↓ka? wa↑taʃiniwa, na↓nika na↑yami↓gotoga a↓ru yo↓oni mi↑eta↓ri ʃi↑ma↓ʃItakedo......
|
15 |
+
dataset/nene/nen001_015.wav|0|---- na↓Nte, i↑Qte mi↓tadakedesUkara, fU↑ka↓i i↑miwaarimase↓Nyo.
|
16 |
+
dataset/nene/nen001_016.wav|0|i↓e, so↑oyuu wa↓kejanakuQte...... ta↓da na↑Ntona↓ku. so↑Nna ki↑ga ʃI↑ta↓dakedesUkara.
|
17 |
+
dataset/nene/nen001_017.wav|0|a, i↓e, ze↑Qtaini hi↑mitsUto i↑u wa↓kejanainode ki↑ni ʃi↑na↓ide ku↑dasa↓i.
|
18 |
+
dataset/nene/nen001_018.wav|0|ko↑okIʃiNdakede ko↑rare↓ruto ko↑ma↓runode, ho↑Ntooni na↑ya↓Nde ru↑niNi↓gainiwa, hi↑romenai yo↓oni o↑negai ʃI↑te i↑rudakedesU.
|
19 |
+
dataset/nene/nen001_019.wav|0|so↑Nna gyo↑ogyooʃi↓i ha↑naʃi↓janaku...... wa↑taʃiwa, o↑kakeNni ʃo↑zoku ʃI↑te i↑ru N↓desU.
|
20 |
+
dataset/nene/nen001_020.wav|0|ha↓i. so↑no o↑kakeNde↓sU.
|
21 |
+
dataset/nene/nen001_021.wav|0|so↑odesUyo. ji↑tsu↓wa wa↑taʃi↓taʧiga nyu↑ugakU su↑ru ma↓ekara a↓Qta bu↑katsuna N↓desU. i↓mawa bu↑iNga i↑nakUte, ʃo↑zoku ʃI↑te↓ru no↑wa wa↑taʃI hi↑to↓ridesUga.
|
22 |
+
dataset/nene/nen001_022.wav|0|na↑node sa↑ikiNwa, ga↑kUsee↓kaino ho↓okarawa ke↓Qkoo tsU↑tsu↓karetete...... ka↑tsudoomo, ha↑Qpyoona↓doo i↑yoku↓tekini ʃI↑te i↑ru wa↑kejaarimase↓Nkarane.
|
23 |
+
dataset/nene/nen001_023.wav|0|i↓e, wa↑taʃiwa u↑ranaio. o↑karutoto i↑Qtemo ha↑bawa hi↑ro↓inode. so↑reni bu↓niwa wa↑taʃIʃi↓ka i↑mase↓Nkara, ke↓Qkoo sU↑ki↓ni de↑ki↓ru N↓desU.
|
24 |
+
dataset/nene/nen001_024.wav|0|sa↑sugani ʃi↑rohebiu↓ranaiwa de↑kimase↓Nyo.
|
25 |
+
dataset/nene/nen001_025.wav|0|i↑ʧioo. ʃi↑Qte ha↑ima↓sUkedo, wa↑taʃini de↑ki↓ru no↑wa ta↑roQtogu↓raidesU. a↑ku↓made ʃu↑mite↓edodesUkara.
|
26 |
+
dataset/nene/nen001_026.wav|0|a↑ku↓made u↑ranaino e↑NʧooseNjoono mo↑no↓desUkara. ji↑Nseeso↓odaNnaNte i↑uhodo ta↓isoona mo↑nojaarimase↓N.
|
27 |
+
dataset/nene/nen001_027.wav|0|de↓mo...... mo↓ʃI ho↓ʃinakuNmo ki↑raidenakere↓ba, i↓tsudemo bu↑ʃItsuni ki↓te ku↑dasa↓i.
|
28 |
+
dataset/nene/nen001_028.wav|0|a, ha↓i. su↓guni i↑kima↓sU.
|
29 |
+
dataset/nene/nen001_029.wav|0|so↑reja↓a wa↑taʃiwa ko↑rede.
|
30 |
+
dataset/nene/nen001_030.wav|0|a↑no, se↑Nse↓e.
|
31 |
+
dataset/nene/nen001_031.wav|0|ha↓i. ki↑zu↓itara ko↑Nna ji↑kaNni na↓Qte ʃi↑ma↓Qte.
|
32 |
+
dataset/nene/nen001_032.wav|0|to↑ʃo↓ʃItsuwa...... mo↓o da↓remo i↑nai N↓desUka?
|
33 |
+
dataset/nene/nen001_033.wav|0|so↑no ma↓eni ʧo↓Qto ʃi↑rabebutsuo sa↑setehoʃii N↓desUga, ka↑gi↓o ka↑ʃIte mo↑raemase↓Nka?
|
34 |
+
dataset/nene/nen001_034.wav|0|ta↑roQtoka↓adono ko↑to↓de ʃo↓oʃoo. ji↑kaNwa ka↑karimase↓N. go↓fuN...... w a mu↓rikamo...... de↓mo ni↑juQ↓puNmo a↑re↓ba o↑warima↓sUkara...... o↑negai ʃi↑ma↓sU.
|
35 |
+
dataset/nene/nen001_035.wav|0|wa↑karima↓ʃIta. a↑ri↓gatoo...... go↑zaima↓sU.
|
36 |
+
dataset/nene/nen001_036.wav|0|w a, ha↓i......... ki↑o tsU↑kema↓sU...... ha↓a, ha↓a......
|
37 |
+
dataset/nene/nen001_037.wav|0|...... ha↓a...... ha↓a......
|
38 |
+
dataset/nene/nen001_038.wav|0|...... da↑ijo↓obu, de↓sUyone...... N↓Q......
|
39 |
+
dataset/nene/nen001_039.wav|0|...... ha↓a...... ha↓a......
|
40 |
+
dataset/nene/nen001_040.wav|0|N↓Q, N↓N......
|
41 |
+
dataset/nene/nen001_041.wav|0|a...... a↓Q, N↓Q...... N↓N N, N↓Q, N↓Q, N↓uuQ......
|
42 |
+
dataset/nene/nen001_042.wav|0|N↓Q, fu↓uu...... ha↓a, ha↓a, a↓Q, a↓aaa...... fa↓aa......
|
43 |
+
dataset/nene/nen001_043.wav|0|ha↓a, ha↓aa...... N, N, N↓Q...... ha↓aa...... a, a, a...... a↓a...... N↓Q......!
|
44 |
+
dataset/nene/nen001_044.wav|0|a↓a, mo↓o...... k o, N↑na n o, sa↑iteede su↓Q...... ha↓a, ha↓a...... ga↑kuiN↓naide, o↑Q, o↓nanii...... o su↑ru↓naNteQ, N, N↓NQ.
|
45 |
+
dataset/nene/nen001_045.wav|0|a↓Q, a↓aaa...... ha↓a, ha↓a, ha↓aaa...... N, N↓NQQ.
|
46 |
+
dataset/nene/nen001_046.wav|0|ha↓a, ha↓a...... wa↑taʃi, na↓Nte ko↑to...... a↓Q, a↓aa...... de↓mo, ki↑moʧiyo↓kUte to↑marimase↓N...... N, N↓NQ, f a, a, a↓aaa......
|
47 |
+
dataset/nene/nen001_047.wav|0|u↓Q...... a, a, a↓Q, a↓Q, hi↓aQ, N, N↓haQ...... ha↓Q, ha↓Q, N↓NuuQ.
|
48 |
+
dataset/nene/nen001_048.wav|0|ha↓a...... ha↓a...... N↓Q, N↓NN...... N↓a, N↓a, a↓Q...... N↓Q, i↓iiQ.
|
49 |
+
dataset/nene/nen001_049.wav|0|N↓Q, N↓Q, N ku↓Q...... hi↓Q, a↓Q, a↓Q, a↓Q, N↓NNNQ......
|
50 |
+
dataset/nene/nen001_050.wav|0|ha↓aaa...... ha↓a, ha↓a...... N↓NQ, N↓Q, N↓NN, N↓a...... N↓a, a, a, a↓aaa...... Q!
|
51 |
+
dataset/nene/nen001_051.wav|0|da↑me, ha↓yaku, ʃi↑naito...... se↑Nse↓ega, mo↑do↓Qte, ki↑ma↓sU...... ko↑Nna to↑koro, mi↓raretara...... N↓Q, N↓Q, N↓a, N↓a, a, a, a, a, a......
|
52 |
+
dataset/nene/nen001_052.wav|0|a, a, N, N↓NN Q...... ha↓Q, ha↓a, ha↓a, a↓aa mo↓o, ho↑Ntooni, sa↑iteede su↓Q......
|
53 |
+
dataset/nene/nen001_053.wav|0|N↓Q...... N↓haaQ, ha↓a, ha↓a...... N↓N, N↓a, N↓aa...... a, a, a↓aa......
|
54 |
+
dataset/nene/nen001_054.wav|0|ha↓aa, ha↓aaaaa, ki↑moʧii↓i...... a↓Q, a↓i, a↓iQ, i↓i...... ho↑Ntooni...... N↓N.
|
55 |
+
dataset/nene/nen001_055.wav|0|ha↓yaku...... ko↑Nna ko↑to, ha↓yaku...... se↑Nse↓ega, ma↑ta yo↑osuo...... mi↑ni, kI↑tariʃi↓nai, u↑ʧini...... ha↓a, ha↓a, ha↓a, ha↓a.
|
56 |
+
dataset/nene/nen001_056.wav|0|ha↓a, ha↓a, ha↓aQ, ha↓aaa...... a, a, a...... a↓a...... a↓aNQ, N↓Q, N↓NQ.
|
57 |
+
dataset/nene/nen001_057.wav|0|N↓NQ, N, N↓haaaaaa...... ha↓a, ha↓a, ha↓aaN, a↓Q, a↑N, a↓aaNQ.
|
58 |
+
dataset/nene/nen001_058.wav|0|ha↓a, ha↓a, ha↓aa...... yo↑dare, de↑ʧau...... ju↑ru...... N↓aQ, ha↓a, ha↓aa...... ha↓Q, ha↓Q, ha↓Q...... ju↑ru.
|
59 |
+
dataset/nene/nen001_059.wav|0|ko↑Nna, n i, ki↑moʧii↓inaNte, sa↑iteede↓sU...... ho↑Ntoo, to↑ʃo↓ʃItsude o↓naniinaNte, sa↑itee su↑gima↓sU...... de↓mo, de↓mo.
|
60 |
+
dataset/nene/nen001_060.wav|0|a↑a↓aaa...... i↓mawa, to↑merarenakUte...... ju↑ru...... ha↓a, ha↓aa...... a↑aaa↓aa......
|
61 |
+
dataset/nene/nen001_061.wav|0|ha↓a, ha↓a, ha↓a...... ha↓yaku...... a↓Q, a↓Q, a↓aa...... a↓aa, ha↓yaku, ha↓yaku.
|
62 |
+
dataset/nene/nen001_062.wav|0|ha↓yaku, i↑ki↓tai, i↑ki↓tai...... ko↑no ma↑maiki↓taii...... ha↓a, ha↓a, ha↓a, a↓a, a↓Q, a↓Q, a↓Q, a↓Q, a↓aQ!
|
63 |
+
dataset/nene/nen001_063.wav|0|a↓Q, a↓Q, a↓aaa...... ʃi↑bire↓te, ki↓ta a...... da↑me, i↓kIsou...... N↓Q, da↑me↓janakUte, a↓Q, a↓Q, ha↓yaku, ko↑no ma↑ma...... ha↓yakUhayakUhayaku.
|
losses.py
ADDED
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import torch
|
2 |
+
from torch.nn import functional as F
|
3 |
+
|
4 |
+
import commons
|
5 |
+
|
6 |
+
|
7 |
+
def feature_loss(fmap_r, fmap_g):
|
8 |
+
loss = 0
|
9 |
+
for dr, dg in zip(fmap_r, fmap_g):
|
10 |
+
for rl, gl in zip(dr, dg):
|
11 |
+
rl = rl.float().detach()
|
12 |
+
gl = gl.float()
|
13 |
+
loss += torch.mean(torch.abs(rl - gl))
|
14 |
+
|
15 |
+
return loss * 2
|
16 |
+
|
17 |
+
|
18 |
+
def discriminator_loss(disc_real_outputs, disc_generated_outputs):
|
19 |
+
loss = 0
|
20 |
+
r_losses = []
|
21 |
+
g_losses = []
|
22 |
+
for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
|
23 |
+
dr = dr.float()
|
24 |
+
dg = dg.float()
|
25 |
+
r_loss = torch.mean((1-dr)**2)
|
26 |
+
g_loss = torch.mean(dg**2)
|
27 |
+
loss += (r_loss + g_loss)
|
28 |
+
r_losses.append(r_loss.item())
|
29 |
+
g_losses.append(g_loss.item())
|
30 |
+
|
31 |
+
return loss, r_losses, g_losses
|
32 |
+
|
33 |
+
|
34 |
+
def generator_loss(disc_outputs):
|
35 |
+
loss = 0
|
36 |
+
gen_losses = []
|
37 |
+
for dg in disc_outputs:
|
38 |
+
dg = dg.float()
|
39 |
+
l = torch.mean((1-dg)**2)
|
40 |
+
gen_losses.append(l)
|
41 |
+
loss += l
|
42 |
+
|
43 |
+
return loss, gen_losses
|
44 |
+
|
45 |
+
|
46 |
+
def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
|
47 |
+
"""
|
48 |
+
z_p, logs_q: [b, h, t_t]
|
49 |
+
m_p, logs_p: [b, h, t_t]
|
50 |
+
"""
|
51 |
+
z_p = z_p.float()
|
52 |
+
logs_q = logs_q.float()
|
53 |
+
m_p = m_p.float()
|
54 |
+
logs_p = logs_p.float()
|
55 |
+
z_mask = z_mask.float()
|
56 |
+
|
57 |
+
kl = logs_p - logs_q - 0.5
|
58 |
+
kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
|
59 |
+
kl = torch.sum(kl * z_mask)
|
60 |
+
l = kl / torch.sum(z_mask)
|
61 |
+
return l
|
mel_processing.py
ADDED
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import os
|
3 |
+
import random
|
4 |
+
import torch
|
5 |
+
from torch import nn
|
6 |
+
import torch.nn.functional as F
|
7 |
+
import torch.utils.data
|
8 |
+
import numpy as np
|
9 |
+
|
10 |
+
import logging
|
11 |
+
|
12 |
+
numba_logger = logging.getLogger('numba')
|
13 |
+
numba_logger.setLevel(logging.WARNING)
|
14 |
+
import warnings
|
15 |
+
warnings.filterwarnings('ignore')
|
16 |
+
import librosa
|
17 |
+
import librosa.util as librosa_util
|
18 |
+
from librosa.util import normalize, pad_center, tiny
|
19 |
+
from scipy.signal import get_window
|
20 |
+
from scipy.io.wavfile import read
|
21 |
+
from librosa.filters import mel as librosa_mel_fn
|
22 |
+
|
23 |
+
MAX_WAV_VALUE = 32768.0
|
24 |
+
|
25 |
+
|
26 |
+
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
|
27 |
+
"""
|
28 |
+
PARAMS
|
29 |
+
------
|
30 |
+
C: compression factor
|
31 |
+
"""
|
32 |
+
return torch.log(torch.clamp(x, min=clip_val) * C)
|
33 |
+
|
34 |
+
|
35 |
+
def dynamic_range_decompression_torch(x, C=1):
|
36 |
+
"""
|
37 |
+
PARAMS
|
38 |
+
------
|
39 |
+
C: compression factor used to compress
|
40 |
+
"""
|
41 |
+
return torch.exp(x) / C
|
42 |
+
|
43 |
+
|
44 |
+
def spectral_normalize_torch(magnitudes):
|
45 |
+
output = dynamic_range_compression_torch(magnitudes)
|
46 |
+
return output
|
47 |
+
|
48 |
+
|
49 |
+
def spectral_de_normalize_torch(magnitudes):
|
50 |
+
output = dynamic_range_decompression_torch(magnitudes)
|
51 |
+
return output
|
52 |
+
|
53 |
+
|
54 |
+
mel_basis = {}
|
55 |
+
hann_window = {}
|
56 |
+
|
57 |
+
|
58 |
+
def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
|
59 |
+
if torch.min(y) < -1.:
|
60 |
+
print('min value is ', torch.min(y))
|
61 |
+
if torch.max(y) > 1.:
|
62 |
+
print('max value is ', torch.max(y))
|
63 |
+
|
64 |
+
global hann_window
|
65 |
+
dtype_device = str(y.dtype) + '_' + str(y.device)
|
66 |
+
wnsize_dtype_device = str(win_size) + '_' + dtype_device
|
67 |
+
if wnsize_dtype_device not in hann_window:
|
68 |
+
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
|
69 |
+
|
70 |
+
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
|
71 |
+
y = y.squeeze(1)
|
72 |
+
|
73 |
+
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
|
74 |
+
center=center, pad_mode='reflect', normalized=False, onesided=True)
|
75 |
+
|
76 |
+
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
|
77 |
+
return spec
|
78 |
+
|
79 |
+
|
80 |
+
def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
|
81 |
+
global mel_basis
|
82 |
+
dtype_device = str(spec.dtype) + '_' + str(spec.device)
|
83 |
+
fmax_dtype_device = str(fmax) + '_' + dtype_device
|
84 |
+
if fmax_dtype_device not in mel_basis:
|
85 |
+
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
|
86 |
+
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device)
|
87 |
+
spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
|
88 |
+
spec = spectral_normalize_torch(spec)
|
89 |
+
return spec
|
90 |
+
|
91 |
+
|
92 |
+
def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
|
93 |
+
if torch.min(y) < -1.:
|
94 |
+
print('min value is ', torch.min(y))
|
95 |
+
if torch.max(y) > 1.:
|
96 |
+
print('max value is ', torch.max(y))
|
97 |
+
|
98 |
+
global mel_basis, hann_window
|
99 |
+
dtype_device = str(y.dtype) + '_' + str(y.device)
|
100 |
+
fmax_dtype_device = str(fmax) + '_' + dtype_device
|
101 |
+
wnsize_dtype_device = str(win_size) + '_' + dtype_device
|
102 |
+
if fmax_dtype_device not in mel_basis:
|
103 |
+
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
|
104 |
+
mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device)
|
105 |
+
if wnsize_dtype_device not in hann_window:
|
106 |
+
hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
|
107 |
+
|
108 |
+
y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
|
109 |
+
y = y.squeeze(1)
|
110 |
+
|
111 |
+
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
|
112 |
+
center=center, pad_mode='reflect', normalized=False, onesided=True)
|
113 |
+
|
114 |
+
spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
|
115 |
+
|
116 |
+
spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
|
117 |
+
spec = spectral_normalize_torch(spec)
|
118 |
+
|
119 |
+
return spec
|
models.py
ADDED
@@ -0,0 +1,537 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|