Core23 commited on
Commit
d926ad2
1 Parent(s): ced805a

Upload 2 files

Browse files
Files changed (2) hide show
  1. data_utils.py +392 -0
  2. train.py +290 -0
data_utils.py ADDED
@@ -0,0 +1,392 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import time
2
+ import os
3
+ import random
4
+ import numpy as np
5
+ import torch
6
+ import torch.utils.data
7
+
8
+ import commons
9
+ from mel_processing import spectrogram_torch
10
+ from utils import load_wav_to_torch, load_filepaths_and_text
11
+ from text import text_to_sequence, cleaned_text_to_sequence
12
+
13
+
14
+ class TextAudioLoader(torch.utils.data.Dataset):
15
+ """
16
+ 1) loads audio, text pairs
17
+ 2) normalizes text and converts them to sequences of integers
18
+ 3) computes spectrograms from audio files.
19
+ """
20
+ def __init__(self, audiopaths_and_text, hparams):
21
+ self.audiopaths_and_text = load_filepaths_and_text(audiopaths_and_text)
22
+ self.text_cleaners = hparams.text_cleaners
23
+ self.max_wav_value = hparams.max_wav_value
24
+ self.sampling_rate = hparams.sampling_rate
25
+ self.filter_length = hparams.filter_length
26
+ self.hop_length = hparams.hop_length
27
+ self.win_length = hparams.win_length
28
+ self.sampling_rate = hparams.sampling_rate
29
+
30
+ self.cleaned_text = getattr(hparams, "cleaned_text", False)
31
+
32
+ self.add_blank = hparams.add_blank
33
+ self.min_text_len = getattr(hparams, "min_text_len", 1)
34
+ self.max_text_len = getattr(hparams, "max_text_len", 190)
35
+
36
+ random.seed(1234)
37
+ random.shuffle(self.audiopaths_and_text)
38
+ self._filter()
39
+
40
+
41
+ def _filter(self):
42
+ """
43
+ Filter text & store spec lengths
44
+ """
45
+ # Store spectrogram lengths for Bucketing
46
+ # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
47
+ # spec_length = wav_length // hop_length
48
+
49
+ audiopaths_and_text_new = []
50
+ lengths = []
51
+ for audiopath, text in self.audiopaths_and_text:
52
+ if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
53
+ audiopaths_and_text_new.append([audiopath, text])
54
+ lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
55
+ self.audiopaths_and_text = audiopaths_and_text_new
56
+ self.lengths = lengths
57
+
58
+ def get_audio_text_pair(self, audiopath_and_text):
59
+ # separate filename and text
60
+ audiopath, text = audiopath_and_text[0], audiopath_and_text[1]
61
+ text = self.get_text(text)
62
+ spec, wav = self.get_audio(audiopath)
63
+ return (text, spec, wav)
64
+
65
+ def get_audio(self, filename):
66
+ audio, sampling_rate = load_wav_to_torch(filename)
67
+ if sampling_rate != self.sampling_rate:
68
+ raise ValueError("{} {} SR doesn't match target {} SR".format(
69
+ sampling_rate, self.sampling_rate))
70
+ audio_norm = audio / self.max_wav_value
71
+ audio_norm = audio_norm.unsqueeze(0)
72
+ spec_filename = filename.replace(".wav", ".spec.pt")
73
+ if os.path.exists(spec_filename):
74
+ spec = torch.load(spec_filename)
75
+ else:
76
+ spec = spectrogram_torch(audio_norm, self.filter_length,
77
+ self.sampling_rate, self.hop_length, self.win_length,
78
+ center=False)
79
+ spec = torch.squeeze(spec, 0)
80
+ torch.save(spec, spec_filename)
81
+ return spec, audio_norm
82
+
83
+ def get_text(self, text):
84
+ if self.cleaned_text:
85
+ text_norm = cleaned_text_to_sequence(text)
86
+ else:
87
+ text_norm = text_to_sequence(text, self.text_cleaners)
88
+ if self.add_blank:
89
+ text_norm = commons.intersperse(text_norm, 0)
90
+ text_norm = torch.LongTensor(text_norm)
91
+ return text_norm
92
+
93
+ def __getitem__(self, index):
94
+ return self.get_audio_text_pair(self.audiopaths_and_text[index])
95
+
96
+ def __len__(self):
97
+ return len(self.audiopaths_and_text)
98
+
99
+
100
+ class TextAudioCollate():
101
+ """ Zero-pads model inputs and targets
102
+ """
103
+ def __init__(self, return_ids=False):
104
+ self.return_ids = return_ids
105
+
106
+ def __call__(self, batch):
107
+ """Collate's training batch from normalized text and aduio
108
+ PARAMS
109
+ ------
110
+ batch: [text_normalized, spec_normalized, wav_normalized]
111
+ """
112
+ # Right zero-pad all one-hot text sequences to max input length
113
+ _, ids_sorted_decreasing = torch.sort(
114
+ torch.LongTensor([x[1].size(1) for x in batch]),
115
+ dim=0, descending=True)
116
+
117
+ max_text_len = max([len(x[0]) for x in batch])
118
+ max_spec_len = max([x[1].size(1) for x in batch])
119
+ max_wav_len = max([x[2].size(1) for x in batch])
120
+
121
+ text_lengths = torch.LongTensor(len(batch))
122
+ spec_lengths = torch.LongTensor(len(batch))
123
+ wav_lengths = torch.LongTensor(len(batch))
124
+
125
+ text_padded = torch.LongTensor(len(batch), max_text_len)
126
+ spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
127
+ wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
128
+ text_padded.zero_()
129
+ spec_padded.zero_()
130
+ wav_padded.zero_()
131
+ for i in range(len(ids_sorted_decreasing)):
132
+ row = batch[ids_sorted_decreasing[i]]
133
+
134
+ text = row[0]
135
+ text_padded[i, :text.size(0)] = text
136
+ text_lengths[i] = text.size(0)
137
+
138
+ spec = row[1]
139
+ spec_padded[i, :, :spec.size(1)] = spec
140
+ spec_lengths[i] = spec.size(1)
141
+
142
+ wav = row[2]
143
+ wav_padded[i, :, :wav.size(1)] = wav
144
+ wav_lengths[i] = wav.size(1)
145
+
146
+ if self.return_ids:
147
+ return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, ids_sorted_decreasing
148
+ return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths
149
+
150
+
151
+ """Multi speaker version"""
152
+ class TextAudioSpeakerLoader(torch.utils.data.Dataset):
153
+ """
154
+ 1) loads audio, speaker_id, text pairs
155
+ 2) normalizes text and converts them to sequences of integers
156
+ 3) computes spectrograms from audio files.
157
+ """
158
+ def __init__(self, audiopaths_sid_text, hparams):
159
+ self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
160
+ self.text_cleaners = hparams.text_cleaners
161
+ self.max_wav_value = hparams.max_wav_value
162
+ self.sampling_rate = hparams.sampling_rate
163
+ self.filter_length = hparams.filter_length
164
+ self.hop_length = hparams.hop_length
165
+ self.win_length = hparams.win_length
166
+ self.sampling_rate = hparams.sampling_rate
167
+
168
+ self.cleaned_text = getattr(hparams, "cleaned_text", False)
169
+
170
+ self.add_blank = hparams.add_blank
171
+ self.min_text_len = getattr(hparams, "min_text_len", 1)
172
+ self.max_text_len = getattr(hparams, "max_text_len", 190)
173
+
174
+ random.seed(1234)
175
+ random.shuffle(self.audiopaths_sid_text)
176
+ self._filter()
177
+
178
+ def _filter(self):
179
+ """
180
+ Filter text & store spec lengths
181
+ """
182
+ # Store spectrogram lengths for Bucketing
183
+ # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
184
+ # spec_length = wav_length // hop_length
185
+
186
+ audiopaths_sid_text_new = []
187
+ lengths = []
188
+ for audiopath, sid, text in self.audiopaths_sid_text:
189
+ if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
190
+ audiopaths_sid_text_new.append([audiopath, sid, text])
191
+ lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
192
+ self.audiopaths_sid_text = audiopaths_sid_text_new
193
+ self.lengths = lengths
194
+
195
+ def get_audio_text_speaker_pair(self, audiopath_sid_text):
196
+ # separate filename, speaker_id and text
197
+ audiopath, sid, text = audiopath_sid_text[0], audiopath_sid_text[1], audiopath_sid_text[2]
198
+ text = self.get_text(text)
199
+ spec, wav = self.get_audio(audiopath)
200
+ sid = self.get_sid(sid)
201
+ return (text, spec, wav, sid)
202
+
203
+ def get_audio(self, filename):
204
+ audio, sampling_rate = load_wav_to_torch(filename)
205
+ if sampling_rate != self.sampling_rate:
206
+ raise ValueError("{} {} SR doesn't match target {} SR".format(
207
+ sampling_rate, self.sampling_rate))
208
+ audio_norm = audio / self.max_wav_value
209
+ audio_norm = audio_norm.unsqueeze(0)
210
+ spec_filename = filename.replace(".wav", ".spec.pt")
211
+ if os.path.exists(spec_filename):
212
+ spec = torch.load(spec_filename)
213
+ else:
214
+ spec = spectrogram_torch(audio_norm, self.filter_length,
215
+ self.sampling_rate, self.hop_length, self.win_length,
216
+ center=False)
217
+ spec = torch.squeeze(spec, 0)
218
+ torch.save(spec, spec_filename)
219
+ return spec, audio_norm
220
+
221
+ def get_text(self, text):
222
+ if self.cleaned_text:
223
+ text_norm = cleaned_text_to_sequence(text)
224
+ else:
225
+ text_norm = text_to_sequence(text, self.text_cleaners)
226
+ if self.add_blank:
227
+ text_norm = commons.intersperse(text_norm, 0)
228
+ text_norm = torch.LongTensor(text_norm)
229
+ return text_norm
230
+
231
+ def get_sid(self, sid):
232
+ sid = torch.LongTensor([int(sid)])
233
+ return sid
234
+
235
+ def __getitem__(self, index):
236
+ return self.get_audio_text_speaker_pair(self.audiopaths_sid_text[index])
237
+
238
+ def __len__(self):
239
+ return len(self.audiopaths_sid_text)
240
+
241
+
242
+ class TextAudioSpeakerCollate():
243
+ """ Zero-pads model inputs and targets
244
+ """
245
+ def __init__(self, return_ids=False):
246
+ self.return_ids = return_ids
247
+
248
+ def __call__(self, batch):
249
+ """Collate's training batch from normalized text, audio and speaker identities
250
+ PARAMS
251
+ ------
252
+ batch: [text_normalized, spec_normalized, wav_normalized, sid]
253
+ """
254
+ # Right zero-pad all one-hot text sequences to max input length
255
+ _, ids_sorted_decreasing = torch.sort(
256
+ torch.LongTensor([x[1].size(1) for x in batch]),
257
+ dim=0, descending=True)
258
+
259
+ max_text_len = max([len(x[0]) for x in batch])
260
+ max_spec_len = max([x[1].size(1) for x in batch])
261
+ max_wav_len = max([x[2].size(1) for x in batch])
262
+
263
+ text_lengths = torch.LongTensor(len(batch))
264
+ spec_lengths = torch.LongTensor(len(batch))
265
+ wav_lengths = torch.LongTensor(len(batch))
266
+ sid = torch.LongTensor(len(batch))
267
+
268
+ text_padded = torch.LongTensor(len(batch), max_text_len)
269
+ spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
270
+ wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
271
+ text_padded.zero_()
272
+ spec_padded.zero_()
273
+ wav_padded.zero_()
274
+ for i in range(len(ids_sorted_decreasing)):
275
+ row = batch[ids_sorted_decreasing[i]]
276
+
277
+ text = row[0]
278
+ text_padded[i, :text.size(0)] = text
279
+ text_lengths[i] = text.size(0)
280
+
281
+ spec = row[1]
282
+ spec_padded[i, :, :spec.size(1)] = spec
283
+ spec_lengths[i] = spec.size(1)
284
+
285
+ wav = row[2]
286
+ wav_padded[i, :, :wav.size(1)] = wav
287
+ wav_lengths[i] = wav.size(1)
288
+
289
+ sid[i] = row[3]
290
+
291
+ if self.return_ids:
292
+ return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, ids_sorted_decreasing
293
+ return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid
294
+
295
+
296
+ class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
297
+ """
298
+ Maintain similar input lengths in a batch.
299
+ Length groups are specified by boundaries.
300
+ Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
301
+
302
+ It removes samples which are not included in the boundaries.
303
+ Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
304
+ """
305
+ def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True):
306
+ super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
307
+ self.lengths = dataset.lengths
308
+ self.batch_size = batch_size
309
+ self.boundaries = boundaries
310
+
311
+ self.buckets, self.num_samples_per_bucket = self._create_buckets()
312
+ self.total_size = sum(self.num_samples_per_bucket)
313
+ self.num_samples = self.total_size // self.num_replicas
314
+
315
+ def _create_buckets(self):
316
+ buckets = [[] for _ in range(len(self.boundaries) - 1)]
317
+ for i in range(len(self.lengths)):
318
+ length = self.lengths[i]
319
+ idx_bucket = self._bisect(length)
320
+ if idx_bucket != -1:
321
+ buckets[idx_bucket].append(i)
322
+
323
+ for i in range(len(buckets) - 1, 0, -1):
324
+ if len(buckets[i]) == 0:
325
+ buckets.pop(i)
326
+ self.boundaries.pop(i+1)
327
+
328
+ num_samples_per_bucket = []
329
+ for i in range(len(buckets)):
330
+ len_bucket = len(buckets[i])
331
+ total_batch_size = self.num_replicas * self.batch_size
332
+ rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size
333
+ num_samples_per_bucket.append(len_bucket + rem)
334
+ return buckets, num_samples_per_bucket
335
+
336
+ def __iter__(self):
337
+ # deterministically shuffle based on epoch
338
+ g = torch.Generator()
339
+ g.manual_seed(self.epoch)
340
+
341
+ indices = []
342
+ if self.shuffle:
343
+ for bucket in self.buckets:
344
+ indices.append(torch.randperm(len(bucket), generator=g).tolist())
345
+ else:
346
+ for bucket in self.buckets:
347
+ indices.append(list(range(len(bucket))))
348
+
349
+ batches = []
350
+ for i in range(len(self.buckets)):
351
+ bucket = self.buckets[i]
352
+ len_bucket = len(bucket)
353
+ ids_bucket = indices[i]
354
+ num_samples_bucket = self.num_samples_per_bucket[i]
355
+
356
+ # add extra samples to make it evenly divisible
357
+ rem = num_samples_bucket - len_bucket
358
+ ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
359
+
360
+ # subsample
361
+ ids_bucket = ids_bucket[self.rank::self.num_replicas]
362
+
363
+ # batching
364
+ for j in range(len(ids_bucket) // self.batch_size):
365
+ batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]]
366
+ batches.append(batch)
367
+
368
+ if self.shuffle:
369
+ batch_ids = torch.randperm(len(batches), generator=g).tolist()
370
+ batches = [batches[i] for i in batch_ids]
371
+ self.batches = batches
372
+
373
+ assert len(self.batches) * self.batch_size == self.num_samples
374
+ return iter(self.batches)
375
+
376
+ def _bisect(self, x, lo=0, hi=None):
377
+ if hi is None:
378
+ hi = len(self.boundaries) - 1
379
+
380
+ if hi > lo:
381
+ mid = (hi + lo) // 2
382
+ if self.boundaries[mid] < x and x <= self.boundaries[mid+1]:
383
+ return mid
384
+ elif x <= self.boundaries[mid]:
385
+ return self._bisect(x, lo, mid)
386
+ else:
387
+ return self._bisect(x, mid + 1, hi)
388
+ else:
389
+ return -1
390
+
391
+ def __len__(self):
392
+ return self.num_samples // self.batch_size
train.py ADDED
@@ -0,0 +1,290 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import argparse
4
+ import itertools
5
+ import math
6
+ import torch
7
+ from torch import nn, optim
8
+ from torch.nn import functional as F
9
+ from torch.utils.data import DataLoader
10
+ from torch.utils.tensorboard import SummaryWriter
11
+ import torch.multiprocessing as mp
12
+ import torch.distributed as dist
13
+ from torch.nn.parallel import DistributedDataParallel as DDP
14
+ from torch.cuda.amp import autocast, GradScaler
15
+
16
+ import commons
17
+ import utils
18
+ from data_utils import (
19
+ TextAudioLoader,
20
+ TextAudioCollate,
21
+ DistributedBucketSampler
22
+ )
23
+ from models import (
24
+ SynthesizerTrn,
25
+ MultiPeriodDiscriminator,
26
+ )
27
+ from losses import (
28
+ generator_loss,
29
+ discriminator_loss,
30
+ feature_loss,
31
+ kl_loss
32
+ )
33
+ from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
34
+ from text.symbols import symbols
35
+
36
+
37
+ torch.backends.cudnn.benchmark = True
38
+ global_step = 0
39
+
40
+
41
+ def main():
42
+ """Assume Single Node Multi GPUs Training Only"""
43
+ assert torch.cuda.is_available(), "CPU training is not allowed."
44
+
45
+ n_gpus = torch.cuda.device_count()
46
+ os.environ['MASTER_ADDR'] = 'localhost'
47
+ os.environ['MASTER_PORT'] = '80000'
48
+
49
+ hps = utils.get_hparams()
50
+ mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
51
+
52
+
53
+ def run(rank, n_gpus, hps):
54
+ global global_step
55
+ if rank == 0:
56
+ logger = utils.get_logger(hps.model_dir)
57
+ logger.info(hps)
58
+ utils.check_git_hash(hps.model_dir)
59
+ writer = SummaryWriter(log_dir=hps.model_dir)
60
+ writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval"))
61
+
62
+ dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank)
63
+ torch.manual_seed(hps.train.seed)
64
+ torch.cuda.set_device(rank)
65
+
66
+ train_dataset = TextAudioLoader(hps.data.training_files, hps.data)
67
+ train_sampler = DistributedBucketSampler(
68
+ train_dataset,
69
+ hps.train.batch_size,
70
+ [32,300,400,500,600,700,800,900,1000],
71
+ num_replicas=n_gpus,
72
+ rank=rank,
73
+ shuffle=True)
74
+ collate_fn = TextAudioCollate()
75
+ train_loader = DataLoader(train_dataset, num_workers=8, shuffle=False, pin_memory=True,
76
+ collate_fn=collate_fn, batch_sampler=train_sampler)
77
+ if rank == 0:
78
+ eval_dataset = TextAudioLoader(hps.data.validation_files, hps.data)
79
+ eval_loader = DataLoader(eval_dataset, num_workers=8, shuffle=False,
80
+ batch_size=hps.train.batch_size, pin_memory=True,
81
+ drop_last=False, collate_fn=collate_fn)
82
+
83
+ net_g = SynthesizerTrn(
84
+ len(symbols),
85
+ hps.data.filter_length // 2 + 1,
86
+ hps.train.segment_size // hps.data.hop_length,
87
+ **hps.model).cuda(rank)
88
+ net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
89
+ optim_g = torch.optim.AdamW(
90
+ net_g.parameters(),
91
+ hps.train.learning_rate,
92
+ betas=hps.train.betas,
93
+ eps=hps.train.eps)
94
+ optim_d = torch.optim.AdamW(
95
+ net_d.parameters(),
96
+ hps.train.learning_rate,
97
+ betas=hps.train.betas,
98
+ eps=hps.train.eps)
99
+ net_g = DDP(net_g, device_ids=[rank])
100
+ net_d = DDP(net_d, device_ids=[rank])
101
+
102
+ try:
103
+ _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g)
104
+ _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d)
105
+ global_step = (epoch_str - 1) * len(train_loader)
106
+ except:
107
+ epoch_str = 1
108
+ global_step = 0
109
+
110
+ scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
111
+ scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
112
+
113
+ scaler = GradScaler(enabled=hps.train.fp16_run)
114
+
115
+ for epoch in range(epoch_str, hps.train.epochs + 1):
116
+ if rank==0:
117
+ train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval])
118
+ else:
119
+ train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None)
120
+ scheduler_g.step()
121
+ scheduler_d.step()
122
+
123
+
124
+ def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers):
125
+ net_g, net_d = nets
126
+ optim_g, optim_d = optims
127
+ scheduler_g, scheduler_d = schedulers
128
+ train_loader, eval_loader = loaders
129
+ if writers is not None:
130
+ writer, writer_eval = writers
131
+
132
+ train_loader.batch_sampler.set_epoch(epoch)
133
+ global global_step
134
+
135
+ net_g.train()
136
+ net_d.train()
137
+ for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(train_loader):
138
+ x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True)
139
+ spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True)
140
+ y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True)
141
+
142
+ with autocast(enabled=hps.train.fp16_run):
143
+ y_hat, l_length, attn, ids_slice, x_mask, z_mask,\
144
+ (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths)
145
+
146
+ mel = spec_to_mel_torch(
147
+ spec,
148
+ hps.data.filter_length,
149
+ hps.data.n_mel_channels,
150
+ hps.data.sampling_rate,
151
+ hps.data.mel_fmin,
152
+ hps.data.mel_fmax)
153
+ y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
154
+ y_hat_mel = mel_spectrogram_torch(
155
+ y_hat.squeeze(1),
156
+ hps.data.filter_length,
157
+ hps.data.n_mel_channels,
158
+ hps.data.sampling_rate,
159
+ hps.data.hop_length,
160
+ hps.data.win_length,
161
+ hps.data.mel_fmin,
162
+ hps.data.mel_fmax
163
+ )
164
+
165
+ y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice
166
+
167
+ # Discriminator
168
+ y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
169
+ with autocast(enabled=False):
170
+ loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g)
171
+ loss_disc_all = loss_disc
172
+ optim_d.zero_grad()
173
+ scaler.scale(loss_disc_all).backward()
174
+ scaler.unscale_(optim_d)
175
+ grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
176
+ scaler.step(optim_d)
177
+
178
+ with autocast(enabled=hps.train.fp16_run):
179
+ # Generator
180
+ y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
181
+ with autocast(enabled=False):
182
+ loss_dur = torch.sum(l_length.float())
183
+ loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
184
+ loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
185
+
186
+ loss_fm = feature_loss(fmap_r, fmap_g)
187
+ loss_gen, losses_gen = generator_loss(y_d_hat_g)
188
+ loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl
189
+ optim_g.zero_grad()
190
+ scaler.scale(loss_gen_all).backward()
191
+ scaler.unscale_(optim_g)
192
+ grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
193
+ scaler.step(optim_g)
194
+ scaler.update()
195
+
196
+ if rank==0:
197
+ if global_step % hps.train.log_interval == 0:
198
+ lr = optim_g.param_groups[0]['lr']
199
+ losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl]
200
+ logger.info('Train Epoch: {} [{:.0f}%]'.format(
201
+ epoch,
202
+ 100. * batch_idx / len(train_loader)))
203
+ logger.info([x.item() for x in losses] + [global_step, lr])
204
+
205
+ scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g}
206
+ scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl})
207
+
208
+ scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)})
209
+ scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)})
210
+ scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)})
211
+ image_dict = {
212
+ "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()),
213
+ "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()),
214
+ "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
215
+ "all/attn": utils.plot_alignment_to_numpy(attn[0,0].data.cpu().numpy())
216
+ }
217
+ utils.summarize(
218
+ writer=writer,
219
+ global_step=global_step,
220
+ images=image_dict,
221
+ scalars=scalar_dict)
222
+
223
+ if global_step % hps.train.eval_interval == 0:
224
+ evaluate(hps, net_g, eval_loader, writer_eval)
225
+ utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step)))
226
+ utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step)))
227
+ global_step += 1
228
+
229
+ if rank == 0:
230
+ logger.info('====> Epoch: {}'.format(epoch))
231
+
232
+
233
+ def evaluate(hps, generator, eval_loader, writer_eval):
234
+ generator.eval()
235
+ with torch.no_grad():
236
+ for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(eval_loader):
237
+ x, x_lengths = x.cuda(0), x_lengths.cuda(0)
238
+ spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0)
239
+ y, y_lengths = y.cuda(0), y_lengths.cuda(0)
240
+
241
+ # remove else
242
+ x = x[:1]
243
+ x_lengths = x_lengths[:1]
244
+ spec = spec[:1]
245
+ spec_lengths = spec_lengths[:1]
246
+ y = y[:1]
247
+ y_lengths = y_lengths[:1]
248
+ break
249
+ y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, max_len=1000)
250
+ y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length
251
+
252
+ mel = spec_to_mel_torch(
253
+ spec,
254
+ hps.data.filter_length,
255
+ hps.data.n_mel_channels,
256
+ hps.data.sampling_rate,
257
+ hps.data.mel_fmin,
258
+ hps.data.mel_fmax)
259
+ y_hat_mel = mel_spectrogram_torch(
260
+ y_hat.squeeze(1).float(),
261
+ hps.data.filter_length,
262
+ hps.data.n_mel_channels,
263
+ hps.data.sampling_rate,
264
+ hps.data.hop_length,
265
+ hps.data.win_length,
266
+ hps.data.mel_fmin,
267
+ hps.data.mel_fmax
268
+ )
269
+ image_dict = {
270
+ "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy())
271
+ }
272
+ audio_dict = {
273
+ "gen/audio": y_hat[0,:,:y_hat_lengths[0]]
274
+ }
275
+ if global_step == 0:
276
+ image_dict.update({"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())})
277
+ audio_dict.update({"gt/audio": y[0,:,:y_lengths[0]]})
278
+
279
+ utils.summarize(
280
+ writer=writer_eval,
281
+ global_step=global_step,
282
+ images=image_dict,
283
+ audios=audio_dict,
284
+ audio_sampling_rate=hps.data.sampling_rate
285
+ )
286
+ generator.train()
287
+
288
+
289
+ if __name__ == "__main__":
290
+ main()