innnky commited on
Commit
e3a7134
1 Parent(s): 2bf872d
G_49200.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da931601660595a5ab841b25668faf97ee66e10d3ae5da03f69c5e61f28476fd
3
+ size 479164585
LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2021 Jaehyeon Kim
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
all_emotions.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48e81667f1fc4ce2b2eaed80fadd0871e1ddfc8933767915954c39ac854d5724
3
+ size 22356096
app.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ import commons
4
+ import utils
5
+ from models import SynthesizerTrn
6
+ from text.symbols import symbols
7
+ from text import text_to_sequence
8
+ import numpy as np
9
+
10
+
11
+ def get_text(text, hps):
12
+ text_norm = text_to_sequence(text, hps.data.text_cleaners)
13
+ if hps.data.add_blank:
14
+ text_norm = commons.intersperse(text_norm, 0)
15
+ text_norm = torch.LongTensor(text_norm)
16
+ return text_norm
17
+ hps = utils.get_hparams_from_file("./configs/vtubers.json")
18
+ net_g = SynthesizerTrn(
19
+ len(symbols),
20
+ hps.data.filter_length // 2 + 1,
21
+ hps.train.segment_size // hps.data.hop_length,
22
+ n_speakers=hps.data.n_speakers,
23
+ **hps.model)
24
+ _ = net_g.eval()
25
+
26
+ _ = utils.load_checkpoint("./G_49200.pth", net_g, None)
27
+ all_emotions = np.load("all_emotions.npy")
28
+ emotion_dict = {
29
+ "小声": 2077,
30
+ "激动": 111,
31
+ "平静1": 434,
32
+ "平静2": 3554
33
+ }
34
+ import random
35
+ def tts(txt, emotion):
36
+ stn_tst = get_text(txt, hps)
37
+ randsample = None
38
+ with torch.no_grad():
39
+ x_tst = stn_tst.unsqueeze(0)
40
+ x_tst_lengths = torch.LongTensor([stn_tst.size(0)])
41
+ sid = torch.LongTensor([0])
42
+ if type(emotion) ==int:
43
+ emo = torch.FloatTensor(all_emotions[emotion]).unsqueeze(0)
44
+ elif emotion == "random":
45
+ emo = torch.randn([1,1024])
46
+ elif emotion == "random_sample":
47
+ randint = random.randint(0, all_emotions.shape[0])
48
+ emo = torch.FloatTensor(all_emotions[randint]).unsqueeze(0)
49
+ randsample = randint
50
+ elif emotion.endswith("wav"):
51
+ import emotion_extract
52
+ emo = torch.FloatTensor(emotion_extract.extract_wav(emotion))
53
+ else:
54
+ emo = torch.FloatTensor(all_emotions[emotion_dict[emotion]]).unsqueeze(0)
55
+
56
+ audio = net_g.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=0.667, noise_scale_w=0.8, length_scale=1, emo=emo)[0][0,0].data.float().numpy()
57
+ return audio, randsample
58
+
59
+
60
+ def tts1(text, emotion):
61
+ if len(text) > 150:
62
+ return "Error: Text is too long", None
63
+ audio, _ = tts(text, emotion)
64
+ return "Success", (hps.data.sampling_rate, audio)
65
+
66
+ def tts2(text):
67
+ if len(text) > 150:
68
+ return "Error: Text is too long", None
69
+ audio, randsample = tts(text, "random_sample")
70
+
71
+ return str(randsample), (hps.data.sampling_rate, audio)
72
+
73
+ def tts3(text, sample):
74
+ if len(text) > 150:
75
+ return "Error: Text is too long", None
76
+ try:
77
+ audio, _ = tts(text, int(sample))
78
+ return "Success", (hps.data.sampling_rate, audio)
79
+ except:
80
+ return "输入参数不为整数或其他错误"
81
+
82
+ app = gr.Blocks()
83
+ with app:
84
+ with gr.Tabs():
85
+ with gr.TabItem("使用预制情感合成"):
86
+ tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
87
+ tts_input2 = gr.Dropdown(label="情感", choices=list(emotion_dict.keys()), value="平静1")
88
+ tts_submit = gr.Button("合成音频", variant="primary")
89
+ tts_output1 = gr.Textbox(label="Message")
90
+ tts_output2 = gr.Audio(label="Output")
91
+ tts_submit.click(tts1, [tts_input1, tts_input2], [tts_output1, tts_output2])
92
+ with gr.TabItem("随机抽取训练集样本作为情感参数"):
93
+ tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
94
+ tts_submit = gr.Button("合成音频", variant="primary")
95
+ tts_output1 = gr.Textbox(label="随机样本id(可用于第三个tab中合成)")
96
+ tts_output2 = gr.Audio(label="Output")
97
+ tts_submit.click(tts2, [tts_input1], [tts_output1, tts_output2])
98
+
99
+ with gr.TabItem("使用情感样本id作为情感参数"):
100
+
101
+ tts_input1 = gr.TextArea(label="日语文本", value="こんにちは。私わあやちねねです。")
102
+ tts_input2 = gr.Number(label="情感样本id", value=2004)
103
+ tts_submit = gr.Button("合成音频", variant="primary")
104
+ tts_output1 = gr.Textbox(label="Message")
105
+ tts_output2 = gr.Audio(label="Output")
106
+ tts_submit.click(tts3, [tts_input1, tts_input2], [tts_output1, tts_output2])
107
+
108
+ with gr.TabItem("使用参考音频作为情感参数"):
109
+ tts_input1 = gr.TextArea(label="text", value="暂未实现")
110
+
111
+ app.launch()
attentions.py ADDED
@@ -0,0 +1,303 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import math
3
+ import numpy as np
4
+ import torch
5
+ from torch import nn
6
+ from torch.nn import functional as F
7
+
8
+ import commons
9
+ import modules
10
+ from modules import LayerNorm
11
+
12
+
13
+ class Encoder(nn.Module):
14
+ def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
15
+ super().__init__()
16
+ self.hidden_channels = hidden_channels
17
+ self.filter_channels = filter_channels
18
+ self.n_heads = n_heads
19
+ self.n_layers = n_layers
20
+ self.kernel_size = kernel_size
21
+ self.p_dropout = p_dropout
22
+ self.window_size = window_size
23
+
24
+ self.drop = nn.Dropout(p_dropout)
25
+ self.attn_layers = nn.ModuleList()
26
+ self.norm_layers_1 = nn.ModuleList()
27
+ self.ffn_layers = nn.ModuleList()
28
+ self.norm_layers_2 = nn.ModuleList()
29
+ for i in range(self.n_layers):
30
+ self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
31
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
32
+ self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
33
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
34
+
35
+ def forward(self, x, x_mask):
36
+ attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
37
+ x = x * x_mask
38
+ for i in range(self.n_layers):
39
+ y = self.attn_layers[i](x, x, attn_mask)
40
+ y = self.drop(y)
41
+ x = self.norm_layers_1[i](x + y)
42
+
43
+ y = self.ffn_layers[i](x, x_mask)
44
+ y = self.drop(y)
45
+ x = self.norm_layers_2[i](x + y)
46
+ x = x * x_mask
47
+ return x
48
+
49
+
50
+ class Decoder(nn.Module):
51
+ def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
52
+ super().__init__()
53
+ self.hidden_channels = hidden_channels
54
+ self.filter_channels = filter_channels
55
+ self.n_heads = n_heads
56
+ self.n_layers = n_layers
57
+ self.kernel_size = kernel_size
58
+ self.p_dropout = p_dropout
59
+ self.proximal_bias = proximal_bias
60
+ self.proximal_init = proximal_init
61
+
62
+ self.drop = nn.Dropout(p_dropout)
63
+ self.self_attn_layers = nn.ModuleList()
64
+ self.norm_layers_0 = nn.ModuleList()
65
+ self.encdec_attn_layers = nn.ModuleList()
66
+ self.norm_layers_1 = nn.ModuleList()
67
+ self.ffn_layers = nn.ModuleList()
68
+ self.norm_layers_2 = nn.ModuleList()
69
+ for i in range(self.n_layers):
70
+ self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
71
+ self.norm_layers_0.append(LayerNorm(hidden_channels))
72
+ self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
73
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
74
+ self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
75
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
76
+
77
+ def forward(self, x, x_mask, h, h_mask):
78
+ """
79
+ x: decoder input
80
+ h: encoder output
81
+ """
82
+ self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
83
+ encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
84
+ x = x * x_mask
85
+ for i in range(self.n_layers):
86
+ y = self.self_attn_layers[i](x, x, self_attn_mask)
87
+ y = self.drop(y)
88
+ x = self.norm_layers_0[i](x + y)
89
+
90
+ y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
91
+ y = self.drop(y)
92
+ x = self.norm_layers_1[i](x + y)
93
+
94
+ y = self.ffn_layers[i](x, x_mask)
95
+ y = self.drop(y)
96
+ x = self.norm_layers_2[i](x + y)
97
+ x = x * x_mask
98
+ return x
99
+
100
+
101
+ class MultiHeadAttention(nn.Module):
102
+ def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
103
+ super().__init__()
104
+ assert channels % n_heads == 0
105
+
106
+ self.channels = channels
107
+ self.out_channels = out_channels
108
+ self.n_heads = n_heads
109
+ self.p_dropout = p_dropout
110
+ self.window_size = window_size
111
+ self.heads_share = heads_share
112
+ self.block_length = block_length
113
+ self.proximal_bias = proximal_bias
114
+ self.proximal_init = proximal_init
115
+ self.attn = None
116
+
117
+ self.k_channels = channels // n_heads
118
+ self.conv_q = nn.Conv1d(channels, channels, 1)
119
+ self.conv_k = nn.Conv1d(channels, channels, 1)
120
+ self.conv_v = nn.Conv1d(channels, channels, 1)
121
+ self.conv_o = nn.Conv1d(channels, out_channels, 1)
122
+ self.drop = nn.Dropout(p_dropout)
123
+
124
+ if window_size is not None:
125
+ n_heads_rel = 1 if heads_share else n_heads
126
+ rel_stddev = self.k_channels**-0.5
127
+ self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
128
+ self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
129
+
130
+ nn.init.xavier_uniform_(self.conv_q.weight)
131
+ nn.init.xavier_uniform_(self.conv_k.weight)
132
+ nn.init.xavier_uniform_(self.conv_v.weight)
133
+ if proximal_init:
134
+ with torch.no_grad():
135
+ self.conv_k.weight.copy_(self.conv_q.weight)
136
+ self.conv_k.bias.copy_(self.conv_q.bias)
137
+
138
+ def forward(self, x, c, attn_mask=None):
139
+ q = self.conv_q(x)
140
+ k = self.conv_k(c)
141
+ v = self.conv_v(c)
142
+
143
+ x, self.attn = self.attention(q, k, v, mask=attn_mask)
144
+
145
+ x = self.conv_o(x)
146
+ return x
147
+
148
+ def attention(self, query, key, value, mask=None):
149
+ # reshape [b, d, t] -> [b, n_h, t, d_k]
150
+ b, d, t_s, t_t = (*key.size(), query.size(2))
151
+ query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
152
+ key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
153
+ value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
154
+
155
+ scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
156
+ if self.window_size is not None:
157
+ assert t_s == t_t, "Relative attention is only available for self-attention."
158
+ key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
159
+ rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
160
+ scores_local = self._relative_position_to_absolute_position(rel_logits)
161
+ scores = scores + scores_local
162
+ if self.proximal_bias:
163
+ assert t_s == t_t, "Proximal bias is only available for self-attention."
164
+ scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
165
+ if mask is not None:
166
+ scores = scores.masked_fill(mask == 0, -1e4)
167
+ if self.block_length is not None:
168
+ assert t_s == t_t, "Local attention is only available for self-attention."
169
+ block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
170
+ scores = scores.masked_fill(block_mask == 0, -1e4)
171
+ p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
172
+ p_attn = self.drop(p_attn)
173
+ output = torch.matmul(p_attn, value)
174
+ if self.window_size is not None:
175
+ relative_weights = self._absolute_position_to_relative_position(p_attn)
176
+ value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
177
+ output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
178
+ output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
179
+ return output, p_attn
180
+
181
+ def _matmul_with_relative_values(self, x, y):
182
+ """
183
+ x: [b, h, l, m]
184
+ y: [h or 1, m, d]
185
+ ret: [b, h, l, d]
186
+ """
187
+ ret = torch.matmul(x, y.unsqueeze(0))
188
+ return ret
189
+
190
+ def _matmul_with_relative_keys(self, x, y):
191
+ """
192
+ x: [b, h, l, d]
193
+ y: [h or 1, m, d]
194
+ ret: [b, h, l, m]
195
+ """
196
+ ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
197
+ return ret
198
+
199
+ def _get_relative_embeddings(self, relative_embeddings, length):
200
+ max_relative_position = 2 * self.window_size + 1
201
+ # Pad first before slice to avoid using cond ops.
202
+ pad_length = max(length - (self.window_size + 1), 0)
203
+ slice_start_position = max((self.window_size + 1) - length, 0)
204
+ slice_end_position = slice_start_position + 2 * length - 1
205
+ if pad_length > 0:
206
+ padded_relative_embeddings = F.pad(
207
+ relative_embeddings,
208
+ commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
209
+ else:
210
+ padded_relative_embeddings = relative_embeddings
211
+ used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
212
+ return used_relative_embeddings
213
+
214
+ def _relative_position_to_absolute_position(self, x):
215
+ """
216
+ x: [b, h, l, 2*l-1]
217
+ ret: [b, h, l, l]
218
+ """
219
+ batch, heads, length, _ = x.size()
220
+ # Concat columns of pad to shift from relative to absolute indexing.
221
+ x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
222
+
223
+ # Concat extra elements so to add up to shape (len+1, 2*len-1).
224
+ x_flat = x.view([batch, heads, length * 2 * length])
225
+ x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
226
+
227
+ # Reshape and slice out the padded elements.
228
+ x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
229
+ return x_final
230
+
231
+ def _absolute_position_to_relative_position(self, x):
232
+ """
233
+ x: [b, h, l, l]
234
+ ret: [b, h, l, 2*l-1]
235
+ """
236
+ batch, heads, length, _ = x.size()
237
+ # padd along column
238
+ x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
239
+ x_flat = x.view([batch, heads, length**2 + length*(length -1)])
240
+ # add 0's in the beginning that will skew the elements after reshape
241
+ x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
242
+ x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
243
+ return x_final
244
+
245
+ def _attention_bias_proximal(self, length):
246
+ """Bias for self-attention to encourage attention to close positions.
247
+ Args:
248
+ length: an integer scalar.
249
+ Returns:
250
+ a Tensor with shape [1, 1, length, length]
251
+ """
252
+ r = torch.arange(length, dtype=torch.float32)
253
+ diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
254
+ return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
255
+
256
+
257
+ class FFN(nn.Module):
258
+ def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
259
+ super().__init__()
260
+ self.in_channels = in_channels
261
+ self.out_channels = out_channels
262
+ self.filter_channels = filter_channels
263
+ self.kernel_size = kernel_size
264
+ self.p_dropout = p_dropout
265
+ self.activation = activation
266
+ self.causal = causal
267
+
268
+ if causal:
269
+ self.padding = self._causal_padding
270
+ else:
271
+ self.padding = self._same_padding
272
+
273
+ self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
274
+ self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
275
+ self.drop = nn.Dropout(p_dropout)
276
+
277
+ def forward(self, x, x_mask):
278
+ x = self.conv_1(self.padding(x * x_mask))
279
+ if self.activation == "gelu":
280
+ x = x * torch.sigmoid(1.702 * x)
281
+ else:
282
+ x = torch.relu(x)
283
+ x = self.drop(x)
284
+ x = self.conv_2(self.padding(x * x_mask))
285
+ return x * x_mask
286
+
287
+ def _causal_padding(self, x):
288
+ if self.kernel_size == 1:
289
+ return x
290
+ pad_l = self.kernel_size - 1
291
+ pad_r = 0
292
+ padding = [[0, 0], [0, 0], [pad_l, pad_r]]
293
+ x = F.pad(x, commons.convert_pad_shape(padding))
294
+ return x
295
+
296
+ def _same_padding(self, x):
297
+ if self.kernel_size == 1:
298
+ return x
299
+ pad_l = (self.kernel_size - 1) // 2
300
+ pad_r = self.kernel_size // 2
301
+ padding = [[0, 0], [0, 0], [pad_l, pad_r]]
302
+ x = F.pad(x, commons.convert_pad_shape(padding))
303
+ return x
commons.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import numpy as np
3
+ import torch
4
+ from torch import nn
5
+ from torch.nn import functional as F
6
+
7
+
8
+ def init_weights(m, mean=0.0, std=0.01):
9
+ classname = m.__class__.__name__
10
+ if classname.find("Conv") != -1:
11
+ m.weight.data.normal_(mean, std)
12
+
13
+
14
+ def get_padding(kernel_size, dilation=1):
15
+ return int((kernel_size*dilation - dilation)/2)
16
+
17
+
18
+ def convert_pad_shape(pad_shape):
19
+ l = pad_shape[::-1]
20
+ pad_shape = [item for sublist in l for item in sublist]
21
+ return pad_shape
22
+
23
+
24
+ def intersperse(lst, item):
25
+ result = [item] * (len(lst) * 2 + 1)
26
+ result[1::2] = lst
27
+ return result
28
+
29
+
30
+ def kl_divergence(m_p, logs_p, m_q, logs_q):
31
+ """KL(P||Q)"""
32
+ kl = (logs_q - logs_p) - 0.5
33
+ kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
34
+ return kl
35
+
36
+
37
+ def rand_gumbel(shape):
38
+ """Sample from the Gumbel distribution, protect from overflows."""
39
+ uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
40
+ return -torch.log(-torch.log(uniform_samples))
41
+
42
+
43
+ def rand_gumbel_like(x):
44
+ g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
45
+ return g
46
+
47
+
48
+ def slice_segments(x, ids_str, segment_size=4):
49
+ ret = torch.zeros_like(x[:, :, :segment_size])
50
+ for i in range(x.size(0)):
51
+ idx_str = ids_str[i]
52
+ idx_end = idx_str + segment_size
53
+ ret[i] = x[i, :, idx_str:idx_end]
54
+ return ret
55
+
56
+
57
+ def rand_slice_segments(x, x_lengths=None, segment_size=4):
58
+ b, d, t = x.size()
59
+ if x_lengths is None:
60
+ x_lengths = t
61
+ ids_str_max = x_lengths - segment_size + 1
62
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
63
+ ret = slice_segments(x, ids_str, segment_size)
64
+ return ret, ids_str
65
+
66
+
67
+ def get_timing_signal_1d(
68
+ length, channels, min_timescale=1.0, max_timescale=1.0e4):
69
+ position = torch.arange(length, dtype=torch.float)
70
+ num_timescales = channels // 2
71
+ log_timescale_increment = (
72
+ math.log(float(max_timescale) / float(min_timescale)) /
73
+ (num_timescales - 1))
74
+ inv_timescales = min_timescale * torch.exp(
75
+ torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
76
+ scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
77
+ signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
78
+ signal = F.pad(signal, [0, 0, 0, channels % 2])
79
+ signal = signal.view(1, channels, length)
80
+ return signal
81
+
82
+
83
+ def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
84
+ b, channels, length = x.size()
85
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
86
+ return x + signal.to(dtype=x.dtype, device=x.device)
87
+
88
+
89
+ def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
90
+ b, channels, length = x.size()
91
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
92
+ return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
93
+
94
+
95
+ def subsequent_mask(length):
96
+ mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
97
+ return mask
98
+
99
+
100
+ @torch.jit.script
101
+ def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
102
+ n_channels_int = n_channels[0]
103
+ in_act = input_a + input_b
104
+ t_act = torch.tanh(in_act[:, :n_channels_int, :])
105
+ s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
106
+ acts = t_act * s_act
107
+ return acts
108
+
109
+
110
+ def convert_pad_shape(pad_shape):
111
+ l = pad_shape[::-1]
112
+ pad_shape = [item for sublist in l for item in sublist]
113
+ return pad_shape
114
+
115
+
116
+ def shift_1d(x):
117
+ x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
118
+ return x
119
+
120
+
121
+ def sequence_mask(length, max_length=None):
122
+ if max_length is None:
123
+ max_length = length.max()
124
+ x = torch.arange(max_length, dtype=length.dtype, device=length.device)
125
+ return x.unsqueeze(0) < length.unsqueeze(1)
126
+
127
+
128
+ def generate_path(duration, mask):
129
+ """
130
+ duration: [b, 1, t_x]
131
+ mask: [b, 1, t_y, t_x]
132
+ """
133
+ device = duration.device
134
+
135
+ b, _, t_y, t_x = mask.shape
136
+ cum_duration = torch.cumsum(duration, -1)
137
+
138
+ cum_duration_flat = cum_duration.view(b * t_x)
139
+ path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
140
+ path = path.view(b, t_x, t_y)
141
+ path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
142
+ path = path.unsqueeze(1).transpose(2,3) * mask
143
+ return path
144
+
145
+
146
+ def clip_grad_value_(parameters, clip_value, norm_type=2):
147
+ if isinstance(parameters, torch.Tensor):
148
+ parameters = [parameters]
149
+ parameters = list(filter(lambda p: p.grad is not None, parameters))
150
+ norm_type = float(norm_type)
151
+ if clip_value is not None:
152
+ clip_value = float(clip_value)
153
+
154
+ total_norm = 0
155
+ for p in parameters:
156
+ param_norm = p.grad.data.norm(norm_type)
157
+ total_norm += param_norm.item() ** norm_type
158
+ if clip_value is not None:
159
+ p.grad.data.clamp_(min=-clip_value, max=clip_value)
160
+ total_norm = total_norm ** (1. / norm_type)
161
+ return total_norm
configs/vtubers.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 200,
4
+ "eval_interval": 400,
5
+ "seed": 1234,
6
+ "epochs": 10000,
7
+ "learning_rate": 2e-4,
8
+ "betas": [0.8, 0.99],
9
+ "eps": 1e-9,
10
+ "batch_size": 8,
11
+ "fp16_run": false,
12
+ "lr_decay": 0.999875,
13
+ "segment_size": 8192,
14
+ "init_lr_ratio": 1,
15
+ "warmup_epochs": 0,
16
+ "c_mel": 45,
17
+ "c_kl": 1.0
18
+ },
19
+ "data": {
20
+ "training_files":"filelists/train.txt.cleaned",
21
+ "validation_files":"filelists/val.txt.cleaned",
22
+ "text_cleaners":["japanese_cleaners"],
23
+ "max_wav_value": 32768.0,
24
+ "sampling_rate": 22050,
25
+ "filter_length": 1024,
26
+ "hop_length": 256,
27
+ "win_length": 1024,
28
+ "n_mel_channels": 80,
29
+ "mel_fmin": 0.0,
30
+ "mel_fmax": null,
31
+ "add_blank": true,
32
+ "n_speakers": 7,
33
+ "cleaned_text": true
34
+ },
35
+ "model": {
36
+ "inter_channels": 192,
37
+ "hidden_channels": 192,
38
+ "filter_channels": 768,
39
+ "n_heads": 2,
40
+ "n_layers": 6,
41
+ "kernel_size": 3,
42
+ "p_dropout": 0.1,
43
+ "resblock": "1",
44
+ "resblock_kernel_sizes": [3,7,11],
45
+ "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
46
+ "upsample_rates": [8,8,2,2],
47
+ "upsample_initial_channel": 512,
48
+ "upsample_kernel_sizes": [16,16,4,4],
49
+ "n_layers_q": 3,
50
+ "use_spectral_norm": false,
51
+ "gin_channels": 256
52
+ }
53
+ }
data_utils.py ADDED
@@ -0,0 +1,261 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ import os
3
+ import random
4
+ import numpy as np
5
+ import torch
6
+ import torch.utils.data
7
+
8
+ import commons
9
+ from mel_processing import spectrogram_torch
10
+ from utils import load_wav_to_torch, load_filepaths_and_text
11
+ from text import text_to_sequence, cleaned_text_to_sequence
12
+
13
+ """Multi speaker version"""
14
+ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
15
+ """
16
+ 1) loads audio, speaker_id, text pairs
17
+ 2) normalizes text and converts them to sequences of integers
18
+ 3) computes spectrograms from audio files.
19
+ """
20
+ def __init__(self, audiopaths_sid_text, hparams):
21
+ self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
22
+ self.text_cleaners = hparams.text_cleaners
23
+ self.max_wav_value = hparams.max_wav_value
24
+ self.sampling_rate = hparams.sampling_rate
25
+ self.filter_length = hparams.filter_length
26
+ self.hop_length = hparams.hop_length
27
+ self.win_length = hparams.win_length
28
+ self.sampling_rate = hparams.sampling_rate
29
+
30
+ self.cleaned_text = getattr(hparams, "cleaned_text", False)
31
+
32
+ self.add_blank = hparams.add_blank
33
+ self.min_text_len = getattr(hparams, "min_text_len", 1)
34
+ self.max_text_len = getattr(hparams, "max_text_len", 190)
35
+
36
+ random.seed(1234)
37
+ random.shuffle(self.audiopaths_sid_text)
38
+ self._filter()
39
+
40
+ def _filter(self):
41
+ """
42
+ Filter text & store spec lengths
43
+ """
44
+ # Store spectrogram lengths for Bucketing
45
+ # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
46
+ # spec_length = wav_length // hop_length
47
+
48
+ audiopaths_sid_text_new = []
49
+ lengths = []
50
+ for audiopath, sid, text in self.audiopaths_sid_text:
51
+ if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
52
+ audiopaths_sid_text_new.append([audiopath, sid, text])
53
+ lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
54
+ self.audiopaths_sid_text = audiopaths_sid_text_new
55
+ self.lengths = lengths
56
+
57
+ def get_audio_text_speaker_pair(self, audiopath_sid_text):
58
+ # separate filename, speaker_id and text
59
+ audiopath, sid, text = audiopath_sid_text[0], audiopath_sid_text[1], audiopath_sid_text[2]
60
+ text = self.get_text(text)
61
+ spec, wav = self.get_audio(audiopath)
62
+ sid = self.get_sid(sid)
63
+ emo = torch.FloatTensor(np.load(audiopath+".emo.npy"))
64
+ return (text, spec, wav, sid, emo)
65
+
66
+ def get_audio(self, filename):
67
+ audio, sampling_rate = load_wav_to_torch(filename)
68
+ if sampling_rate != self.sampling_rate:
69
+ raise ValueError("{} {} SR doesn't match target {} SR".format(
70
+ sampling_rate, self.sampling_rate))
71
+ audio_norm = audio / self.max_wav_value
72
+ audio_norm = audio_norm.unsqueeze(0)
73
+ spec_filename = filename.replace(".wav", ".spec.pt")
74
+ if os.path.exists(spec_filename):
75
+ spec = torch.load(spec_filename)
76
+ else:
77
+ spec = spectrogram_torch(audio_norm, self.filter_length,
78
+ self.sampling_rate, self.hop_length, self.win_length,
79
+ center=False)
80
+ spec = torch.squeeze(spec, 0)
81
+ torch.save(spec, spec_filename)
82
+ return spec, audio_norm
83
+
84
+ def get_text(self, text):
85
+ if self.cleaned_text:
86
+ text_norm = cleaned_text_to_sequence(text)
87
+ else:
88
+ text_norm = text_to_sequence(text, self.text_cleaners)
89
+ if self.add_blank:
90
+ text_norm = commons.intersperse(text_norm, 0)
91
+ text_norm = torch.LongTensor(text_norm)
92
+ return text_norm
93
+
94
+ def get_sid(self, sid):
95
+ sid = torch.LongTensor([int(sid)])
96
+ return sid
97
+
98
+ def __getitem__(self, index):
99
+ return self.get_audio_text_speaker_pair(self.audiopaths_sid_text[index])
100
+
101
+ def __len__(self):
102
+ return len(self.audiopaths_sid_text)
103
+
104
+
105
+ class TextAudioSpeakerCollate():
106
+ """ Zero-pads model inputs and targets
107
+ """
108
+ def __init__(self, return_ids=False):
109
+ self.return_ids = return_ids
110
+
111
+ def __call__(self, batch):
112
+ """Collate's training batch from normalized text, audio and speaker identities
113
+ PARAMS
114
+ ------
115
+ batch: [text_normalized, spec_normalized, wav_normalized, sid]
116
+ """
117
+ # Right zero-pad all one-hot text sequences to max input length
118
+ _, ids_sorted_decreasing = torch.sort(
119
+ torch.LongTensor([x[1].size(1) for x in batch]),
120
+ dim=0, descending=True)
121
+
122
+ max_text_len = max([len(x[0]) for x in batch])
123
+ max_spec_len = max([x[1].size(1) for x in batch])
124
+ max_wav_len = max([x[2].size(1) for x in batch])
125
+
126
+ text_lengths = torch.LongTensor(len(batch))
127
+ spec_lengths = torch.LongTensor(len(batch))
128
+ wav_lengths = torch.LongTensor(len(batch))
129
+ sid = torch.LongTensor(len(batch))
130
+
131
+ text_padded = torch.LongTensor(len(batch), max_text_len)
132
+ spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
133
+ wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
134
+ emo = torch.FloatTensor(len(batch), 1024)
135
+
136
+ text_padded.zero_()
137
+ spec_padded.zero_()
138
+ wav_padded.zero_()
139
+ emo.zero_()
140
+
141
+ for i in range(len(ids_sorted_decreasing)):
142
+ row = batch[ids_sorted_decreasing[i]]
143
+
144
+ text = row[0]
145
+ text_padded[i, :text.size(0)] = text
146
+ text_lengths[i] = text.size(0)
147
+
148
+ spec = row[1]
149
+ spec_padded[i, :, :spec.size(1)] = spec
150
+ spec_lengths[i] = spec.size(1)
151
+
152
+ wav = row[2]
153
+ wav_padded[i, :, :wav.size(1)] = wav
154
+ wav_lengths[i] = wav.size(1)
155
+
156
+ sid[i] = row[3]
157
+
158
+ emo[i, :] = row[4]
159
+
160
+ if self.return_ids:
161
+ return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, ids_sorted_decreasing
162
+ return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid,emo
163
+
164
+
165
+ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
166
+ """
167
+ Maintain similar input lengths in a batch.
168
+ Length groups are specified by boundaries.
169
+ Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
170
+
171
+ It removes samples which are not included in the boundaries.
172
+ Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
173
+ """
174
+ def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True):
175
+ super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
176
+ self.lengths = dataset.lengths
177
+ self.batch_size = batch_size
178
+ self.boundaries = boundaries
179
+
180
+ self.buckets, self.num_samples_per_bucket = self._create_buckets()
181
+ self.total_size = sum(self.num_samples_per_bucket)
182
+ self.num_samples = self.total_size // self.num_replicas
183
+
184
+ def _create_buckets(self):
185
+ buckets = [[] for _ in range(len(self.boundaries) - 1)]
186
+ for i in range(len(self.lengths)):
187
+ length = self.lengths[i]
188
+ idx_bucket = self._bisect(length)
189
+ if idx_bucket != -1:
190
+ buckets[idx_bucket].append(i)
191
+
192
+ for i in range(len(buckets) - 1, 0, -1):
193
+ if len(buckets[i]) == 0:
194
+ buckets.pop(i)
195
+ self.boundaries.pop(i+1)
196
+
197
+ num_samples_per_bucket = []
198
+ for i in range(len(buckets)):
199
+ len_bucket = len(buckets[i])
200
+ total_batch_size = self.num_replicas * self.batch_size
201
+ rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size
202
+ num_samples_per_bucket.append(len_bucket + rem)
203
+ return buckets, num_samples_per_bucket
204
+
205
+ def __iter__(self):
206
+ # deterministically shuffle based on epoch
207
+ g = torch.Generator()
208
+ g.manual_seed(self.epoch)
209
+
210
+ indices = []
211
+ if self.shuffle:
212
+ for bucket in self.buckets:
213
+ indices.append(torch.randperm(len(bucket), generator=g).tolist())
214
+ else:
215
+ for bucket in self.buckets:
216
+ indices.append(list(range(len(bucket))))
217
+
218
+ batches = []
219
+ for i in range(len(self.buckets)):
220
+ bucket = self.buckets[i]
221
+ len_bucket = len(bucket)
222
+ ids_bucket = indices[i]
223
+ num_samples_bucket = self.num_samples_per_bucket[i]
224
+
225
+ # add extra samples to make it evenly divisible
226
+ rem = num_samples_bucket - len_bucket
227
+ ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
228
+
229
+ # subsample
230
+ ids_bucket = ids_bucket[self.rank::self.num_replicas]
231
+
232
+ # batching
233
+ for j in range(len(ids_bucket) // self.batch_size):
234
+ batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]]
235
+ batches.append(batch)
236
+
237
+ if self.shuffle:
238
+ batch_ids = torch.randperm(len(batches), generator=g).tolist()
239
+ batches = [batches[i] for i in batch_ids]
240
+ self.batches = batches
241
+
242
+ assert len(self.batches) * self.batch_size == self.num_samples
243
+ return iter(self.batches)
244
+
245
+ def _bisect(self, x, lo=0, hi=None):
246
+ if hi is None:
247
+ hi = len(self.boundaries) - 1
248
+
249
+ if hi > lo:
250
+ mid = (hi + lo) // 2
251
+ if self.boundaries[mid] < x and x <= self.boundaries[mid+1]:
252
+ return mid
253
+ elif x <= self.boundaries[mid]:
254
+ return self._bisect(x, lo, mid)
255
+ else:
256
+ return self._bisect(x, mid + 1, hi)
257
+ else:
258
+ return -1
259
+
260
+ def __len__(self):
261
+ return self.num_samples // self.batch_size
emotion_extract.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from transformers import Wav2Vec2Processor
4
+ from transformers.models.wav2vec2.modeling_wav2vec2 import (
5
+ Wav2Vec2Model,
6
+ Wav2Vec2PreTrainedModel,
7
+ )
8
+ import os
9
+ import librosa
10
+ import numpy as np
11
+
12
+
13
+ class RegressionHead(nn.Module):
14
+ r"""Classification head."""
15
+
16
+ def __init__(self, config):
17
+ super().__init__()
18
+
19
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
20
+ self.dropout = nn.Dropout(config.final_dropout)
21
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
22
+
23
+ def forward(self, features, **kwargs):
24
+ x = features
25
+ x = self.dropout(x)
26
+ x = self.dense(x)
27
+ x = torch.tanh(x)
28
+ x = self.dropout(x)
29
+ x = self.out_proj(x)
30
+
31
+ return x
32
+
33
+
34
+ class EmotionModel(Wav2Vec2PreTrainedModel):
35
+ r"""Speech emotion classifier."""
36
+
37
+ def __init__(self, config):
38
+ super().__init__(config)
39
+
40
+ self.config = config
41
+ self.wav2vec2 = Wav2Vec2Model(config)
42
+ self.classifier = RegressionHead(config)
43
+ self.init_weights()
44
+
45
+ def forward(
46
+ self,
47
+ input_values,
48
+ ):
49
+ outputs = self.wav2vec2(input_values)
50
+ hidden_states = outputs[0]
51
+ hidden_states = torch.mean(hidden_states, dim=1)
52
+ logits = self.classifier(hidden_states)
53
+
54
+ return hidden_states, logits
55
+
56
+
57
+ # load model from hub
58
+ device = 'cuda' if torch.cuda.is_available() else "cpu"
59
+ model_name = 'audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'
60
+ processor = Wav2Vec2Processor.from_pretrained(model_name)
61
+ model = EmotionModel.from_pretrained(model_name).to(device)
62
+
63
+
64
+ def process_func(
65
+ x: np.ndarray,
66
+ sampling_rate: int,
67
+ embeddings: bool = False,
68
+ ) -> np.ndarray:
69
+ r"""Predict emotions or extract embeddings from raw audio signal."""
70
+
71
+ # run through processor to normalize signal
72
+ # always returns a batch, so we just get the first entry
73
+ # then we put it on the device
74
+ y = processor(x, sampling_rate=sampling_rate)
75
+ y = y['input_values'][0]
76
+ y = torch.from_numpy(y).to(device)
77
+
78
+ # run through model
79
+ with torch.no_grad():
80
+ y = model(y)[0 if embeddings else 1]
81
+
82
+ # convert to numpy
83
+ y = y.detach().cpu().numpy()
84
+
85
+ return y
86
+ #
87
+ #
88
+ # def disp(rootpath, wavname):
89
+ # wav, sr = librosa.load(f"{rootpath}/{wavname}", 16000)
90
+ # display(ipd.Audio(wav, rate=sr))
91
+
92
+ rootpath = "dataset/nene"
93
+ embs = []
94
+ wavnames = []
95
+ def extract_dir(path):
96
+ rootpath = path
97
+ for idx, wavname in enumerate(os.listdir(rootpath)):
98
+ wav, sr =librosa.load(f"{rootpath}/{wavname}", 16000)
99
+ emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
100
+ embs.append(emb)
101
+ wavnames.append(wavname)
102
+ np.save(f"{rootpath}/{wavname}.emo.npy", emb.squeeze(0))
103
+ print(idx, wavname)
104
+
105
+ def extract_wav(path):
106
+ wav, sr = librosa.load(path, 16000)
107
+ emb = process_func(np.expand_dims(wav, 0), sr, embeddings=True)
108
+ return emb
109
+
110
+ if __name__ == '__main__':
111
+ for spk in ["serena", "koni", "nyaru","shanoa", "mana"]:
112
+ extract_dir(f"dataset/{spk}")
filelists/train.txt ADDED
The diff for this file is too large to render. See raw diff
 
filelists/train.txt.cleaned ADDED
The diff for this file is too large to render. See raw diff
 
filelists/val.txt ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset/nene/nen001_001.wav|0|はい?呼びました?
2
+ dataset/nene/nen001_002.wav|0|驚かせたならごめんなさい。通りかかったときにちょうど名前が聞こえてきたので
3
+ dataset/nene/nen001_003.wav|0|私に何か用ですか?
4
+ dataset/nene/nen001_004.wav|0|いえ、少し気になっただけですから、別に怒っているわけじゃないです。気にしないで下さい
5
+ dataset/nene/nen001_005.wav|0|それより仮屋さん、例の件ですが――
6
+ dataset/nene/nen001_006.wav|0|喜んでもらえたならなによりです
7
+ dataset/nene/nen001_007.wav|0|また何かあったら、いつでも部室に来て下さい
8
+ dataset/nene/nen001_008.wav|0|大したことじゃないので。それより体調の方はもういいんですか?
9
+ dataset/nene/nen001_009.wav|0|それはよかったです。他に困ったことはありますか?
10
+ dataset/nene/nen001_010.wav|0|何かあったらいつでも話して下さい。学院のことじゃなく、私事に関することでも何でも
11
+ dataset/nene/nen001_011.wav|0|はい、いつでもどうぞ
12
+ dataset/nene/nen001_012.wav|0|保科君も
13
+ dataset/nene/nen001_013.wav|0|もし何か困ったことがあれば、力になりますから
14
+ dataset/nene/nen001_014.wav|0|そうですか?私には、なにか悩み事があるように見えたりしましたけど……
15
+ dataset/nene/nen001_015.wav|0|――なんて、言ってみただけですから、深い意味はありませんよ
16
+ dataset/nene/nen001_016.wav|0|いえ、そういうわけじゃなくって……ただ何となく。そんな気がしただけですから
17
+ dataset/nene/nen001_017.wav|0|あ、いえ、絶対に秘密というわけじゃないので気にしないで下さい
18
+ dataset/nene/nen001_018.wav|0|好奇心だけで来られると困るので、本当に悩んでる人以外には、広めないようにお願いしているだけです
19
+ dataset/nene/nen001_019.wav|0|そんな仰々しい話じゃなく……私は、オカ研に所属しているんです
20
+ dataset/nene/nen001_020.wav|0|はい。そのオカ研です
21
+ dataset/nene/nen001_021.wav|0|そうですよ。実は私たちが入学する前からあった部活なんです。今は部員がいなくて、所属してるのは私一人ですが
22
+ dataset/nene/nen001_022.wav|0|なので最近は、学生会の方からは結構つつかれてて……活動も、発表などを意欲的にしているわけじゃありませんからね
23
+ dataset/nene/nen001_023.wav|0|いえ、私は占いを。オカルトと言っても幅は広いので。それに部には私しかいませんから、結構好きにできるんです
24
+ dataset/nene/nen001_024.wav|0|さすがに白蛇占いはできませんよ
25
+ dataset/nene/nen001_025.wav|0|一応。知ってはいますけど、私にできるのはタロットぐらいです。あくまで趣味程度ですから
26
+ dataset/nene/nen001_026.wav|0|あくまで占いの延長線上のものですから。人生相談なんていうほど大層な物じゃありません
27
+ dataset/nene/nen001_027.wav|0|でも……もし保科君も嫌いでなければ、いつでも部室に来て下さい
28
+ dataset/nene/nen001_028.wav|0|あ、はい。すぐに行きます
29
+ dataset/nene/nen001_029.wav|0|それじゃあ私はこれで
30
+ dataset/nene/nen001_030.wav|0|あの、先生
31
+ dataset/nene/nen001_031.wav|0|はい。気付いたらこんな時間になってしまって
32
+ dataset/nene/nen001_032.wav|0|図書室は……もう誰もいないんですか?
33
+ dataset/nene/nen001_033.wav|0|その前にちょっと調べ物をさせて欲しいんですが、鍵を貸してもらえませんか?
34
+ dataset/nene/nen001_034.wav|0|タロットカードのことで少々。時間はかかりません。5分……は無理かも……でも20分もあれば終わりますから……お願いします
35
+ dataset/nene/nen001_035.wav|0|わかりました。ありがとう……ございます
36
+ dataset/nene/nen001_036.wav|0|は、はい………気をつけます……ハァ、ハァ……
37
+ dataset/nene/nen001_037.wav|0|……ハァ……ハァ……
38
+ dataset/nene/nen001_038.wav|0|……大丈夫、ですよね……んっ……
39
+ dataset/nene/nen001_039.wav|0|……ハァ……ハァ……
40
+ dataset/nene/nen001_040.wav|0|んっ、んん……
41
+ dataset/nene/nen001_041.wav|0|あ……あっ、んっ……んんン、んッ、んッ、んぅぅッ……
42
+ dataset/nene/nen001_042.wav|0|んっ、ふぅぅ……はぁ、はぁ、あっ、ああぁぁ……ふぁぁぁ……
43
+ dataset/nene/nen001_043.wav|0|はぁ、はぁぁ……ん、ん、んっ……はぁぁ……ぁ、ぁ、ぁ……ぁぁ……ンッ……!
44
+ dataset/nene/nen001_044.wav|0|あぁ、もう……こ、んなの、最低ですッ……はぁ、はぁ……学院内で、おっ、オナニー……をするなんてっ、ん、んんっ
45
+ dataset/nene/nen001_045.wav|0|あっ、ああぁぁ……はぁ、はぁ、はぁぁぁ……ん、んンッッ
46
+ dataset/nene/nen001_046.wav|0|はぁ、はぁ……私、��んてこと……あっ、あぁぁ……でも、気持ちよくて止まりません……ん、んんッ、ふぁ、あ、あぁぁぁ……
47
+ dataset/nene/nen001_047.wav|0|うっ……あ、あ、あっ、あっ、ひあっ、ん、んはっ……はっ、はっ、んんぅぅッ
48
+ dataset/nene/nen001_048.wav|0|はぁ……はぁ……んっ、んんん……んぁ、んぁ、あっ……んっ、ぃぃぃッ
49
+ dataset/nene/nen001_049.wav|0|んっ、んっ、んくっ……ひっ、あっ、ぁっ、ぁっ、んんーーッ……
50
+ dataset/nene/nen001_050.wav|0|はぁぁぁ……はぁ、はぁ……んんっ、んっ、んんん、んぁ……んぁ、ぁ、ぁ、ぁぁぁぁ……ッ!
51
+ dataset/nene/nen001_051.wav|0|だめ、早く、しないと……先生が、戻って、きます……こんなところ、見られたら……んっ、んっ、んぁ、んぁ、あ、あ、あ、あ、あ……
52
+ dataset/nene/nen001_052.wav|0|あ、あ、ん、んんんッ……はっ、はぁ、はぁ、あぁぁもう、本当に、最低ですっ……
53
+ dataset/nene/nen001_053.wav|0|んッ……んはぁッ、はぁ、はぁ……ンン、んぁ、んぁぁ……あ、あ、あぁぁ……
54
+ dataset/nene/nen001_054.wav|0|はぁー、はぁーーぁぁ、気持ちいい……あっ、あぃ、あぃっ、いい……本当に……んん
55
+ dataset/nene/nen001_055.wav|0|早く……こんなこと、早く……先生が、また様子を……見に、きたりしない、内に……はぁ、はぁ、はぁ、はぁ
56
+ dataset/nene/nen001_056.wav|0|はぁ、はぁ、はぁっ、はぁぁぁ……ぁ、ぁ、ぁ……ぁぁ……ぁぁんッ、んっ、んーッ
57
+ dataset/nene/nen001_057.wav|0|んんッ、ん、んはぁぁぁーー……はぁ、はぁ、はぁァン、あッ、あん、あぁぁンッ
58
+ dataset/nene/nen001_058.wav|0|はぁ、はぁ、はぁぁ……ヨダレ、でちゃう……じゅる……んぁっ、はぁ、はぁぁ……はっ、はっ、はっ……じゅる
59
+ dataset/nene/nen001_059.wav|0|こんな、に、気持ちいいなんて、最低です……本当、図書室でオナニーなんて、最低すぎます……でも、でも
60
+ dataset/nene/nen001_060.wav|0|あああぁぁ……今は、止められなくて……じゅる……はぁ、はぁぁ……あぁぁぁあぁ……
61
+ dataset/nene/nen001_061.wav|0|はぁ、はぁ、はぁ……早く……あっ、あっ、あぁぁ……ぁぁぁ、早く、早く
62
+ dataset/nene/nen001_062.wav|0|早く、イきたい、イきたい……このままイきたいぃ……はぁ、はぁ、はぁ、ぁぁ、ぁっ、ぁっ、あっ、あっ、あぁッ!
63
+ dataset/nene/nen001_063.wav|0|あっ、あっ、ああぁぁ……痺れて、きたぁ……だめ、イきそう……んっ、だめじゃなくて、あっ、あっ、早く、このまま……早く早く早く
filelists/val.txt.cleaned ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset/nene/nen001_001.wav|0|ha↓i? yo↑bima↓ʃIta?
2
+ dataset/nene/nen001_002.wav|0|o↑doro↓kasetanara go↑meNnasa↓i. to↑orikaka↓Qta to↓kini ʧo↑odo na↑maega kI↑koete ki↓tanode.
3
+ dataset/nene/nen001_003.wav|0|wa↑taʃini na↑nika↓yoodesUka?
4
+ dataset/nene/nen001_004.wav|0|i↓e, sU↑ko↓ʃI ki↑ni na↓QtadakedesUkara, be↑tsuni o↑ko↓Qte i↑ru wa↓kejanaidesU. ki↑ni ʃi↑na↓ide ku↑dasa↓i.
5
+ dataset/nene/nen001_005.wav|0|so↑reyo↓ri ka↑riyasaN, re↓eno ke↓NdesUga----
6
+ dataset/nene/nen001_006.wav|0|yo↑roko↓Nde mo↑raeta↓nara na↓niyoridesU.
7
+ dataset/nene/nen001_007.wav|0|ma↑ta na↓nika a↓Qtara, i↓tsudemo bu↑ʃItsuni ki↓te ku↑dasa↓i.
8
+ dataset/nene/nen001_008.wav|0|ta↓iʃIta ko↑to↓janainode. so↑reyo↓ri ta↑iʧoono ho↓owa mo↓o i↓i N↓desUka?
9
+ dataset/nene/nen001_009.wav|0|so↑rewa yo↓kaQtadesU. ta↓ni ko↑ma↓Qta ko↑to↓wa a↑rima↓sUka?
10
+ dataset/nene/nen001_010.wav|0|na↓nika a↓Qtara i↓tsudemo ha↑na↓ʃIte ku↑dasa↓i. ga↑kuiNno ko↑to↓janaku, ʃi↓jini ka↑Nsu↓ru ko↑to↓demo na↓nidemo.
11
+ dataset/nene/nen001_011.wav|0|ha↓i, i↓tsudemo do↓ozo.
12
+ dataset/nene/nen001_012.wav|0|ho↓ʃinakuNmo.
13
+ dataset/nene/nen001_013.wav|0|mo↓ʃi na↓nika ko↑ma↓Qta ko↑to↓ga a↑re↓ba, ʧI↑kara↓ni na↑rima↓sUkara.
14
+ dataset/nene/nen001_014.wav|0|so↑odesU↓ka? wa↑taʃiniwa, na↓nika na↑yami↓gotoga a↓ru yo↓oni mi↑eta↓ri ʃi↑ma↓ʃItakedo......
15
+ dataset/nene/nen001_015.wav|0|---- na↓Nte, i↑Qte mi↓tadakedesUkara, fU↑ka↓i i↑miwaarimase↓Nyo.
16
+ dataset/nene/nen001_016.wav|0|i↓e, so↑oyuu wa↓kejanakuQte...... ta↓da na↑Ntona↓ku. so↑Nna ki↑ga ʃI↑ta↓dakedesUkara.
17
+ dataset/nene/nen001_017.wav|0|a, i↓e, ze↑Qtaini hi↑mitsUto i↑u wa↓kejanainode ki↑ni ʃi↑na↓ide ku↑dasa↓i.
18
+ dataset/nene/nen001_018.wav|0|ko↑okIʃiNdakede ko↑rare↓ruto ko↑ma↓runode, ho↑Ntooni na↑ya↓Nde ru↑niNi↓gainiwa, hi↑romenai yo↓oni o↑negai ʃI↑te i↑rudakedesU.
19
+ dataset/nene/nen001_019.wav|0|so↑Nna gyo↑ogyooʃi↓i ha↑naʃi↓janaku...... wa↑taʃiwa, o↑kakeNni ʃo↑zoku ʃI↑te i↑ru N↓desU.
20
+ dataset/nene/nen001_020.wav|0|ha↓i. so↑no o↑kakeNde↓sU.
21
+ dataset/nene/nen001_021.wav|0|so↑odesUyo. ji↑tsu↓wa wa↑taʃi↓taʧiga nyu↑ugakU su↑ru ma↓ekara a↓Qta bu↑katsuna N↓desU. i↓mawa bu↑iNga i↑nakUte, ʃo↑zoku ʃI↑te↓ru no↑wa wa↑taʃI hi↑to↓ridesUga.
22
+ dataset/nene/nen001_022.wav|0|na↑node sa↑ikiNwa, ga↑kUsee↓kaino ho↓okarawa ke↓Qkoo tsU↑tsu↓karetete...... ka↑tsudoomo, ha↑Qpyoona↓doo i↑yoku↓tekini ʃI↑te i↑ru wa↑kejaarimase↓Nkarane.
23
+ dataset/nene/nen001_023.wav|0|i↓e, wa↑taʃiwa u↑ranaio. o↑karutoto i↑Qtemo ha↑bawa hi↑ro↓inode. so↑reni bu↓niwa wa↑taʃIʃi↓ka i↑mase↓Nkara, ke↓Qkoo sU↑ki↓ni de↑ki↓ru N↓desU.
24
+ dataset/nene/nen001_024.wav|0|sa↑sugani ʃi↑rohebiu↓ranaiwa de↑kimase↓Nyo.
25
+ dataset/nene/nen001_025.wav|0|i↑ʧioo. ʃi↑Qte ha↑ima↓sUkedo, wa↑taʃini de↑ki↓ru no↑wa ta↑roQtogu↓raidesU. a↑ku↓made ʃu↑mite↓edodesUkara.
26
+ dataset/nene/nen001_026.wav|0|a↑ku↓made u↑ranaino e↑NʧooseNjoono mo↑no↓desUkara. ji↑Nseeso↓odaNnaNte i↑uhodo ta↓isoona mo↑nojaarimase↓N.
27
+ dataset/nene/nen001_027.wav|0|de↓mo...... mo↓ʃI ho↓ʃinakuNmo ki↑raidenakere↓ba, i↓tsudemo bu↑ʃItsuni ki↓te ku↑dasa↓i.
28
+ dataset/nene/nen001_028.wav|0|a, ha↓i. su↓guni i↑kima↓sU.
29
+ dataset/nene/nen001_029.wav|0|so↑reja↓a wa↑taʃiwa ko↑rede.
30
+ dataset/nene/nen001_030.wav|0|a↑no, se↑Nse↓e.
31
+ dataset/nene/nen001_031.wav|0|ha↓i. ki↑zu↓itara ko↑Nna ji↑kaNni na↓Qte ʃi↑ma↓Qte.
32
+ dataset/nene/nen001_032.wav|0|to↑ʃo↓ʃItsuwa...... mo↓o da↓remo i↑nai N↓desUka?
33
+ dataset/nene/nen001_033.wav|0|so↑no ma↓eni ʧo↓Qto ʃi↑rabebutsuo sa↑setehoʃii N↓desUga, ka↑gi↓o ka↑ʃIte mo↑raemase↓Nka?
34
+ dataset/nene/nen001_034.wav|0|ta↑roQtoka↓adono ko↑to↓de ʃo↓oʃoo. ji↑kaNwa ka↑karimase↓N. go↓fuN...... w a mu↓rikamo...... de↓mo ni↑juQ↓puNmo a↑re↓ba o↑warima↓sUkara...... o↑negai ʃi↑ma↓sU.
35
+ dataset/nene/nen001_035.wav|0|wa↑karima↓ʃIta. a↑ri↓gatoo...... go↑zaima↓sU.
36
+ dataset/nene/nen001_036.wav|0|w a, ha↓i......... ki↑o tsU↑kema↓sU...... ha↓a, ha↓a......
37
+ dataset/nene/nen001_037.wav|0|...... ha↓a...... ha↓a......
38
+ dataset/nene/nen001_038.wav|0|...... da↑ijo↓obu, de↓sUyone...... N↓Q......
39
+ dataset/nene/nen001_039.wav|0|...... ha↓a...... ha↓a......
40
+ dataset/nene/nen001_040.wav|0|N↓Q, N↓N......
41
+ dataset/nene/nen001_041.wav|0|a...... a↓Q, N↓Q...... N↓N N, N↓Q, N↓Q, N↓uuQ......
42
+ dataset/nene/nen001_042.wav|0|N↓Q, fu↓uu...... ha↓a, ha↓a, a↓Q, a↓aaa...... fa↓aa......
43
+ dataset/nene/nen001_043.wav|0|ha↓a, ha↓aa...... N, N, N↓Q...... ha↓aa...... a, a, a...... a↓a...... N↓Q......!
44
+ dataset/nene/nen001_044.wav|0|a↓a, mo↓o...... k o, N↑na n o, sa↑iteede su↓Q...... ha↓a, ha↓a...... ga↑kuiN↓naide, o↑Q, o↓nanii...... o su↑ru↓naNteQ, N, N↓NQ.
45
+ dataset/nene/nen001_045.wav|0|a↓Q, a↓aaa...... ha↓a, ha↓a, ha↓aaa...... N, N↓NQQ.
46
+ dataset/nene/nen001_046.wav|0|ha↓a, ha↓a...... wa↑taʃi, na↓Nte ko↑to...... a↓Q, a↓aa...... de↓mo, ki↑moʧiyo↓kUte to↑marimase↓N...... N, N↓NQ, f a, a, a↓aaa......
47
+ dataset/nene/nen001_047.wav|0|u↓Q...... a, a, a↓Q, a↓Q, hi↓aQ, N, N↓haQ...... ha↓Q, ha↓Q, N↓NuuQ.
48
+ dataset/nene/nen001_048.wav|0|ha↓a...... ha↓a...... N↓Q, N↓NN...... N↓a, N↓a, a↓Q...... N↓Q, i↓iiQ.
49
+ dataset/nene/nen001_049.wav|0|N↓Q, N↓Q, N ku↓Q...... hi↓Q, a↓Q, a↓Q, a↓Q, N↓NNNQ......
50
+ dataset/nene/nen001_050.wav|0|ha↓aaa...... ha↓a, ha↓a...... N↓NQ, N↓Q, N↓NN, N↓a...... N↓a, a, a, a↓aaa...... Q!
51
+ dataset/nene/nen001_051.wav|0|da↑me, ha↓yaku, ʃi↑naito...... se↑Nse↓ega, mo↑do↓Qte, ki↑ma↓sU...... ko↑Nna to↑koro, mi↓raretara...... N↓Q, N↓Q, N↓a, N↓a, a, a, a, a, a......
52
+ dataset/nene/nen001_052.wav|0|a, a, N, N↓NN Q...... ha↓Q, ha↓a, ha↓a, a↓aa mo↓o, ho↑Ntooni, sa↑iteede su↓Q......
53
+ dataset/nene/nen001_053.wav|0|N↓Q...... N↓haaQ, ha↓a, ha↓a...... N↓N, N↓a, N↓aa...... a, a, a↓aa......
54
+ dataset/nene/nen001_054.wav|0|ha↓aa, ha↓aaaaa, ki↑moʧii↓i...... a↓Q, a↓i, a↓iQ, i↓i...... ho↑Ntooni...... N↓N.
55
+ dataset/nene/nen001_055.wav|0|ha↓yaku...... ko↑Nna ko↑to, ha↓yaku...... se↑Nse↓ega, ma↑ta yo↑osuo...... mi↑ni, kI↑tariʃi↓nai, u↑ʧini...... ha↓a, ha↓a, ha↓a, ha↓a.
56
+ dataset/nene/nen001_056.wav|0|ha↓a, ha↓a, ha↓aQ, ha↓aaa...... a, a, a...... a↓a...... a↓aNQ, N↓Q, N↓NQ.
57
+ dataset/nene/nen001_057.wav|0|N↓NQ, N, N↓haaaaaa...... ha↓a, ha↓a, ha↓aaN, a↓Q, a↑N, a↓aaNQ.
58
+ dataset/nene/nen001_058.wav|0|ha↓a, ha↓a, ha↓aa...... yo↑dare, de↑ʧau...... ju↑ru...... N↓aQ, ha↓a, ha↓aa...... ha↓Q, ha↓Q, ha↓Q...... ju↑ru.
59
+ dataset/nene/nen001_059.wav|0|ko↑Nna, n i, ki↑moʧii↓inaNte, sa↑iteede↓sU...... ho↑Ntoo, to↑ʃo↓ʃItsude o↓naniinaNte, sa↑itee su↑gima↓sU...... de↓mo, de↓mo.
60
+ dataset/nene/nen001_060.wav|0|a↑a↓aaa...... i↓mawa, to↑merarenakUte...... ju↑ru...... ha↓a, ha↓aa...... a↑aaa↓aa......
61
+ dataset/nene/nen001_061.wav|0|ha↓a, ha↓a, ha↓a...... ha↓yaku...... a↓Q, a↓Q, a↓aa...... a↓aa, ha↓yaku, ha↓yaku.
62
+ dataset/nene/nen001_062.wav|0|ha↓yaku, i↑ki↓tai, i↑ki↓tai...... ko↑no ma↑maiki↓taii...... ha↓a, ha↓a, ha↓a, a↓a, a↓Q, a↓Q, a↓Q, a↓Q, a↓aQ!
63
+ dataset/nene/nen001_063.wav|0|a↓Q, a↓Q, a↓aaa...... ʃi↑bire↓te, ki↓ta a...... da↑me, i↓kIsou...... N↓Q, da↑me↓janakUte, a↓Q, a↓Q, ha↓yaku, ko↑no ma↑ma...... ha↓yakUhayakUhayaku.
losses.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from torch.nn import functional as F
3
+
4
+ import commons
5
+
6
+
7
+ def feature_loss(fmap_r, fmap_g):
8
+ loss = 0
9
+ for dr, dg in zip(fmap_r, fmap_g):
10
+ for rl, gl in zip(dr, dg):
11
+ rl = rl.float().detach()
12
+ gl = gl.float()
13
+ loss += torch.mean(torch.abs(rl - gl))
14
+
15
+ return loss * 2
16
+
17
+
18
+ def discriminator_loss(disc_real_outputs, disc_generated_outputs):
19
+ loss = 0
20
+ r_losses = []
21
+ g_losses = []
22
+ for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
23
+ dr = dr.float()
24
+ dg = dg.float()
25
+ r_loss = torch.mean((1-dr)**2)
26
+ g_loss = torch.mean(dg**2)
27
+ loss += (r_loss + g_loss)
28
+ r_losses.append(r_loss.item())
29
+ g_losses.append(g_loss.item())
30
+
31
+ return loss, r_losses, g_losses
32
+
33
+
34
+ def generator_loss(disc_outputs):
35
+ loss = 0
36
+ gen_losses = []
37
+ for dg in disc_outputs:
38
+ dg = dg.float()
39
+ l = torch.mean((1-dg)**2)
40
+ gen_losses.append(l)
41
+ loss += l
42
+
43
+ return loss, gen_losses
44
+
45
+
46
+ def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
47
+ """
48
+ z_p, logs_q: [b, h, t_t]
49
+ m_p, logs_p: [b, h, t_t]
50
+ """
51
+ z_p = z_p.float()
52
+ logs_q = logs_q.float()
53
+ m_p = m_p.float()
54
+ logs_p = logs_p.float()
55
+ z_mask = z_mask.float()
56
+
57
+ kl = logs_p - logs_q - 0.5
58
+ kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
59
+ kl = torch.sum(kl * z_mask)
60
+ l = kl / torch.sum(z_mask)
61
+ return l
mel_processing.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+ import random
4
+ import torch
5
+ from torch import nn
6
+ import torch.nn.functional as F
7
+ import torch.utils.data
8
+ import numpy as np
9
+
10
+ import logging
11
+
12
+ numba_logger = logging.getLogger('numba')
13
+ numba_logger.setLevel(logging.WARNING)
14
+ import warnings
15
+ warnings.filterwarnings('ignore')
16
+ import librosa
17
+ import librosa.util as librosa_util
18
+ from librosa.util import normalize, pad_center, tiny
19
+ from scipy.signal import get_window
20
+ from scipy.io.wavfile import read
21
+ from librosa.filters import mel as librosa_mel_fn
22
+
23
+ MAX_WAV_VALUE = 32768.0
24
+
25
+
26
+ def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
27
+ """
28
+ PARAMS
29
+ ------
30
+ C: compression factor
31
+ """
32
+ return torch.log(torch.clamp(x, min=clip_val) * C)
33
+
34
+
35
+ def dynamic_range_decompression_torch(x, C=1):
36
+ """
37
+ PARAMS
38
+ ------
39
+ C: compression factor used to compress
40
+ """
41
+ return torch.exp(x) / C
42
+
43
+
44
+ def spectral_normalize_torch(magnitudes):
45
+ output = dynamic_range_compression_torch(magnitudes)
46
+ return output
47
+
48
+
49
+ def spectral_de_normalize_torch(magnitudes):
50
+ output = dynamic_range_decompression_torch(magnitudes)
51
+ return output
52
+
53
+
54
+ mel_basis = {}
55
+ hann_window = {}
56
+
57
+
58
+ def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
59
+ if torch.min(y) < -1.:
60
+ print('min value is ', torch.min(y))
61
+ if torch.max(y) > 1.:
62
+ print('max value is ', torch.max(y))
63
+
64
+ global hann_window
65
+ dtype_device = str(y.dtype) + '_' + str(y.device)
66
+ wnsize_dtype_device = str(win_size) + '_' + dtype_device
67
+ if wnsize_dtype_device not in hann_window:
68
+ hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
69
+
70
+ y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
71
+ y = y.squeeze(1)
72
+
73
+ spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
74
+ center=center, pad_mode='reflect', normalized=False, onesided=True)
75
+
76
+ spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
77
+ return spec
78
+
79
+
80
+ def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
81
+ global mel_basis
82
+ dtype_device = str(spec.dtype) + '_' + str(spec.device)
83
+ fmax_dtype_device = str(fmax) + '_' + dtype_device
84
+ if fmax_dtype_device not in mel_basis:
85
+ mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
86
+ mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device)
87
+ spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
88
+ spec = spectral_normalize_torch(spec)
89
+ return spec
90
+
91
+
92
+ def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
93
+ if torch.min(y) < -1.:
94
+ print('min value is ', torch.min(y))
95
+ if torch.max(y) > 1.:
96
+ print('max value is ', torch.max(y))
97
+
98
+ global mel_basis, hann_window
99
+ dtype_device = str(y.dtype) + '_' + str(y.device)
100
+ fmax_dtype_device = str(fmax) + '_' + dtype_device
101
+ wnsize_dtype_device = str(win_size) + '_' + dtype_device
102
+ if fmax_dtype_device not in mel_basis:
103
+ mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
104
+ mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device)
105
+ if wnsize_dtype_device not in hann_window:
106
+ hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
107
+
108
+ y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
109
+ y = y.squeeze(1)
110
+
111
+ spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
112
+ center=center, pad_mode='reflect', normalized=False, onesided=True)
113
+
114
+ spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
115
+
116
+ spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
117
+ spec = spectral_normalize_torch(spec)
118
+
119
+ return spec
models.py ADDED
@@ -0,0 +1,537 @@