|
2024-11-26 18:49:04,056 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-26 18:49:04,056 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-26 18:49:10,922 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-26 18:49:10,922 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-26 18:49:10,955 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-26 18:49:11,063 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-26 18:49:11,254 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-26 18:49:31,917 example INFO Train Epoch: 1 [0%] |
|
2024-11-26 18:49:31,917 example INFO [2.541557550430298, 3.0494136810302734, 10.05517864227295, 32.88372802734375, 4.077049255371094, 4.26006555557251, 0, 0.0003] |
|
2024-11-26 18:49:39,742 example INFO Saving model and optimizer state at iteration 1 to ./logs/example/G_0.pth |
|
2024-11-26 18:49:40,696 example INFO Saving model and optimizer state at iteration 1 to ./logs/example/D_0.pth |
|
2024-11-26 18:49:41,200 example INFO Saving model and optimizer state at iteration 1 to ./logs/example/DUR_0.pth |
|
2024-11-26 19:08:34,982 example INFO Train Epoch: 1 [48%] |
|
2024-11-26 19:08:34,982 example INFO [2.4109578132629395, 2.5706396102905273, 8.777633666992188, 19.76093292236328, 2.2422327995300293, 1.933606743812561, 200, 0.0003] |
|
2024-11-26 19:18:11,399 example INFO Train Epoch: 1 [96%] |
|
2024-11-26 19:18:11,399 example INFO [2.5435876846313477, 2.448261022567749, 9.059673309326172, 19.583398818969727, 2.237337112426758, 2.405848979949951, 400, 0.0003] |
|
2024-11-26 19:18:41,312 example INFO ====> Epoch: 1 |
|
2024-11-26 19:24:00,985 example INFO Train Epoch: 2 [45%] |
|
2024-11-26 19:24:00,985 example INFO [2.647618055343628, 2.087707042694092, 7.908282279968262, 18.531972885131836, 2.2376270294189453, 1.9061672687530518, 600, 0.0002999625] |
|
2024-11-26 19:29:22,633 example INFO Train Epoch: 2 [93%] |
|
2024-11-26 19:29:22,633 example INFO [2.5348284244537354, 2.308347463607788, 8.462638854980469, 18.91036033630371, 2.3122615814208984, 2.5512633323669434, 800, 0.0002999625] |
|
2024-11-26 19:30:06,102 example INFO ====> Epoch: 2 |
|
2024-11-26 19:34:07,630 example INFO Train Epoch: 3 [41%] |
|
2024-11-26 19:34:07,630 example INFO [2.420340061187744, 2.306619167327881, 7.5471367835998535, 19.336742401123047, 1.937631368637085, 1.5983541011810303, 1000, 0.0002999250046875] |
|
2024-11-26 19:34:11,023 example INFO Saving model and optimizer state at iteration 3 to ./logs/example/G_1000.pth |
|
2024-11-26 19:34:11,910 example INFO Saving model and optimizer state at iteration 3 to ./logs/example/D_1000.pth |
|
2024-11-26 19:34:12,409 example INFO Saving model and optimizer state at iteration 3 to ./logs/example/DUR_1000.pth |
|
2024-11-26 19:38:57,147 example INFO Train Epoch: 3 [89%] |
|
2024-11-26 19:38:57,147 example INFO [2.585174798965454, 2.3677117824554443, 7.019065856933594, 18.167993545532227, 2.2178499698638916, 2.0183539390563965, 1200, 0.0002999250046875] |
|
2024-11-26 19:40:05,776 example INFO ====> Epoch: 3 |
|
2024-11-26 19:44:05,655 example INFO Train Epoch: 4 [37%] |
|
2024-11-26 19:44:05,656 example INFO [2.273341178894043, 2.4962210655212402, 8.339351654052734, 18.80166244506836, 2.0555062294006348, 1.9727874994277954, 1400, 0.00029988751406191406] |
|
2024-11-26 19:48:57,385 example INFO Train Epoch: 4 [86%] |
|
2024-11-26 19:48:57,386 example INFO [2.5784101486206055, 2.212251663208008, 7.54019021987915, 19.040401458740234, 2.1006577014923096, 1.960435390472412, 1600, 0.00029988751406191406] |
|
2024-11-26 19:50:21,317 example INFO ====> Epoch: 4 |
|
2024-11-26 19:53:46,092 example INFO Train Epoch: 5 [34%] |
|
2024-11-26 19:53:46,093 example INFO [2.565809726715088, 2.420555830001831, 5.379451274871826, 17.044912338256836, 2.2584710121154785, 2.0707008838653564, 1800, 0.00029985002812265633] |
|
2024-11-26 19:58:34,875 example INFO Train Epoch: 5 [82%] |
|
2024-11-26 19:58:34,875 example INFO [2.212043046951294, 2.7192039489746094, 9.557937622070312, 18.112442016601562, 2.1680784225463867, 2.2679622173309326, 2000, 0.00029985002812265633] |
|
2024-11-26 19:58:38,780 example INFO Saving model and optimizer state at iteration 5 to ./logs/example/G_2000.pth |
|
2024-11-26 19:58:39,661 example INFO Saving model and optimizer state at iteration 5 to ./logs/example/D_2000.pth |
|
2024-11-26 19:58:40,203 example INFO Saving model and optimizer state at iteration 5 to ./logs/example/DUR_2000.pth |
|
2024-11-26 20:00:20,927 example INFO ====> Epoch: 5 |
|
2024-11-26 20:03:10,474 example INFO Train Epoch: 6 [30%] |
|
2024-11-26 20:03:10,474 example INFO [2.3224828243255615, 2.4680228233337402, 9.063647270202637, 18.80310821533203, 1.9913671016693115, 2.033817768096924, 2200, 0.000299812546869141] |
|
2024-11-26 20:07:53,676 example INFO Train Epoch: 6 [78%] |
|
2024-11-26 20:07:53,676 example INFO [2.5226216316223145, 2.5989973545074463, 8.127426147460938, 17.836816787719727, 1.9180933237075806, 2.358004570007324, 2400, 0.000299812546869141] |
|
2024-11-26 20:09:51,806 example INFO ====> Epoch: 6 |
|
2024-11-26 20:12:10,753 example INFO Train Epoch: 7 [27%] |
|
2024-11-26 20:12:10,754 example INFO [2.477473735809326, 2.7541427612304688, 7.678103923797607, 18.3265438079834, 2.0368576049804688, 2.3729255199432373, 2600, 0.00029977507030078235] |
|
2024-11-26 20:16:13,028 example INFO Train Epoch: 7 [75%] |
|
2024-11-26 20:16:13,029 example INFO [2.301762342453003, 2.5924813747406006, 8.40649127960205, 18.336811065673828, 1.953812837600708, 2.401137590408325, 2800, 0.00029977507030078235] |
|
2024-11-26 20:18:29,163 example INFO ====> Epoch: 7 |
|
2024-11-26 20:20:52,017 example INFO Train Epoch: 8 [23%] |
|
2024-11-26 20:20:52,018 example INFO [2.169847249984741, 2.7582240104675293, 9.405250549316406, 18.719472885131836, 2.2185325622558594, 2.0607266426086426, 3000, 0.00029973759841699473] |
|
2024-11-26 20:20:55,634 example INFO Saving model and optimizer state at iteration 8 to ./logs/example/G_3000.pth |
|
2024-11-26 20:20:56,460 example INFO Saving model and optimizer state at iteration 8 to ./logs/example/D_3000.pth |
|
2024-11-26 20:20:56,967 example INFO Saving model and optimizer state at iteration 8 to ./logs/example/DUR_3000.pth |
|
2024-11-26 20:20:57,061 example INFO .. Free up space by deleting ckpt ./logs/example/D_1000.pth |
|
2024-11-26 20:25:12,157 example INFO Train Epoch: 8 [71%] |
|
2024-11-26 20:25:12,158 example INFO [2.525437831878662, 2.4389328956604004, 7.928208827972412, 18.49370574951172, 2.0713350772857666, 2.23592472076416, 3200, 0.00029973759841699473] |
|
2024-11-26 20:27:37,185 example INFO ====> Epoch: 8 |
|
2024-11-26 20:29:24,613 example INFO Train Epoch: 9 [19%] |
|
2024-11-26 20:29:24,613 example INFO [2.6366963386535645, 2.1804654598236084, 8.095945358276367, 18.787744522094727, 2.0685019493103027, 1.6909383535385132, 3400, 0.00029970013121719257] |
|
2024-11-26 20:33:31,401 example INFO Train Epoch: 9 [67%] |
|
2024-11-26 20:33:31,402 example INFO [2.2302088737487793, 2.6564509868621826, 7.5971150398254395, 18.229063034057617, 2.107734203338623, 2.2073583602905273, 3600, 0.00029970013121719257] |
|
2024-11-26 20:36:39,259 example INFO ====> Epoch: 9 |
|
2024-11-26 20:38:20,807 example INFO Train Epoch: 10 [16%] |
|
2024-11-26 20:38:20,807 example INFO [2.3652050495147705, 2.7580788135528564, 8.969311714172363, 18.311193466186523, 1.894762635231018, 2.143314838409424, 3800, 0.0002996626687007904] |
|
2024-11-26 20:42:18,002 example INFO Train Epoch: 10 [64%] |
|
2024-11-26 20:42:18,003 example INFO [2.5669913291931152, 2.8428924083709717, 8.066770553588867, 17.977642059326172, 2.277568817138672, 1.777411699295044, 4000, 0.0002996626687007904] |
|
2024-11-26 20:42:21,004 example INFO Saving model and optimizer state at iteration 10 to ./logs/example/G_4000.pth |
|
2024-11-26 20:42:21,884 example INFO Saving model and optimizer state at iteration 10 to ./logs/example/D_4000.pth |
|
2024-11-26 20:42:22,393 example INFO Saving model and optimizer state at iteration 10 to ./logs/example/DUR_4000.pth |
|
2024-11-26 20:42:22,404 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_1000.pth |
|
2024-11-26 20:42:22,486 example INFO .. Free up space by deleting ckpt ./logs/example/D_2000.pth |
|
2024-11-26 20:45:29,298 example INFO ====> Epoch: 10 |
|
2024-11-26 20:46:43,363 example INFO Train Epoch: 11 [12%] |
|
2024-11-26 20:46:43,363 example INFO [2.5637621879577637, 2.4507176876068115, 8.063015937805176, 18.25080108642578, 2.1173360347747803, 1.9061498641967773, 4200, 0.0002996252108672028] |
|
2024-11-26 20:51:55,604 example INFO Train Epoch: 11 [60%] |
|
2024-11-26 20:51:55,604 example INFO [2.374271869659424, 2.4040138721466064, 7.142162799835205, 18.116920471191406, 2.1122097969055176, 1.8840633630752563, 4400, 0.0002996252108672028] |
|
2024-11-26 20:55:51,229 example INFO ====> Epoch: 11 |
|
2024-11-26 20:56:44,911 example INFO Train Epoch: 12 [8%] |
|
2024-11-26 20:56:44,911 example INFO [2.4236879348754883, 2.584141731262207, 7.072590351104736, 17.092369079589844, 2.0170323848724365, 2.105797052383423, 4600, 0.00029958775771584436] |
|
2024-11-26 21:00:47,960 example INFO Train Epoch: 12 [57%] |
|
2024-11-26 21:00:47,961 example INFO [2.1385273933410645, 2.5899105072021484, 9.601863861083984, 18.578054428100586, 2.218010187149048, 2.2040982246398926, 4800, 0.00029958775771584436] |
|
2024-11-26 21:04:29,067 example INFO ====> Epoch: 12 |
|
2024-11-26 21:05:01,146 example INFO Train Epoch: 13 [5%] |
|
2024-11-26 21:05:01,147 example INFO [2.1417975425720215, 2.8775904178619385, 8.302937507629395, 18.755006790161133, 1.9180223941802979, 1.9005039930343628, 5000, 0.0002995503092461299] |
|
2024-11-26 21:05:04,308 example INFO Saving model and optimizer state at iteration 13 to ./logs/example/G_5000.pth |
|
2024-11-26 21:05:05,130 example INFO Saving model and optimizer state at iteration 13 to ./logs/example/D_5000.pth |
|
2024-11-26 21:05:05,634 example INFO Saving model and optimizer state at iteration 13 to ./logs/example/DUR_5000.pth |
|
2024-11-26 21:05:05,645 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_2000.pth |
|
2024-11-26 21:05:05,728 example INFO .. Free up space by deleting ckpt ./logs/example/D_3000.pth |
|
2024-11-26 21:10:18,379 example INFO Train Epoch: 13 [53%] |
|
2024-11-26 21:10:18,379 example INFO [2.6769213676452637, 2.45857572555542, 5.839945316314697, 17.452117919921875, 2.168516159057617, 2.371295690536499, 5200, 0.0002995503092461299] |
|
2024-11-26 21:15:21,484 example INFO ====> Epoch: 13 |
|
2024-11-26 21:15:31,754 example INFO Train Epoch: 14 [1%] |
|
2024-11-26 21:15:31,755 example INFO [2.2612364292144775, 2.3471388816833496, 7.352994918823242, 18.207361221313477, 2.0575942993164062, 2.454235553741455, 5400, 0.0002995128654574741] |
|
2024-11-26 21:20:04,334 example INFO Train Epoch: 14 [49%] |
|
2024-11-26 21:20:04,335 example INFO [2.370386838912964, 2.6557986736297607, 7.399454593658447, 17.9141788482666, 1.96409273147583, 1.697926640510559, 5600, 0.0002995128654574741] |
|
2024-11-26 21:24:52,179 example INFO Train Epoch: 14 [98%] |
|
2024-11-26 21:24:52,179 example INFO [2.415526866912842, 2.259002447128296, 8.938506126403809, 17.732797622680664, 1.9945776462554932, 1.9275295734405518, 5800, 0.0002995128654574741] |
|
2024-11-26 21:25:03,767 example INFO ====> Epoch: 14 |
|
2024-11-26 21:29:34,994 example INFO Train Epoch: 15 [46%] |
|
2024-11-26 21:29:34,994 example INFO [2.4210915565490723, 2.4225971698760986, 8.235459327697754, 17.435447692871094, 1.963327407836914, 1.8453476428985596, 6000, 0.0002994754263492919] |
|
2024-11-26 21:29:38,441 example INFO Saving model and optimizer state at iteration 15 to ./logs/example/G_6000.pth |
|
2024-11-26 21:29:39,273 example INFO Saving model and optimizer state at iteration 15 to ./logs/example/D_6000.pth |
|
2024-11-26 21:29:39,776 example INFO Saving model and optimizer state at iteration 15 to ./logs/example/DUR_6000.pth |
|
2024-11-26 21:29:39,879 example INFO .. Free up space by deleting ckpt ./logs/example/G_1000.pth |
|
2024-11-26 21:29:39,880 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_3000.pth |
|
2024-11-26 21:29:39,967 example INFO .. Free up space by deleting ckpt ./logs/example/D_4000.pth |
|
2024-11-26 21:32:21,398 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-26 21:32:21,398 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-26 21:32:28,070 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-26 21:32:28,070 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-26 21:32:28,102 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-26 21:32:28,211 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-26 21:32:28,318 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-26 21:32:28,321 example INFO Loaded checkpoint './logs/example/DUR_6000.pth' (iteration 15) |
|
2024-11-26 21:32:28,685 example INFO Loaded checkpoint './logs/example/G_6000.pth' (iteration 15) |
|
2024-11-26 21:32:28,809 example INFO Loaded checkpoint './logs/example/D_6000.pth' (iteration 15) |
|
2024-11-26 21:49:39,325 example INFO Train Epoch: 15 [46%] |
|
2024-11-26 21:49:39,326 example INFO [2.414727210998535, 2.4394962787628174, 8.300931930541992, 18.47859001159668, 1.9899643659591675, 2.0701956748962402, 6000, 0.0002999625] |
|
2024-11-26 21:49:47,326 example INFO Saving model and optimizer state at iteration 15 to ./logs/example/G_6000.pth |
|
2024-11-26 21:49:48,491 example INFO Saving model and optimizer state at iteration 15 to ./logs/example/D_6000.pth |
|
2024-11-26 21:49:49,222 example INFO Saving model and optimizer state at iteration 15 to ./logs/example/DUR_6000.pth |
|
2024-11-26 21:59:17,393 example INFO Train Epoch: 15 [94%] |
|
2024-11-26 21:59:17,393 example INFO [2.293100357055664, 2.6375491619110107, 6.996551036834717, 17.70345687866211, 1.993189811706543, 2.222928524017334, 6200, 0.0002999625] |
|
2024-11-26 22:00:08,004 example INFO ====> Epoch: 15 |
|
2024-11-26 22:05:18,748 example INFO Train Epoch: 16 [42%] |
|
2024-11-26 22:05:18,749 example INFO [2.2186038494110107, 2.8117947578430176, 8.053207397460938, 17.514657974243164, 2.104217052459717, 2.3248910903930664, 6400, 0.0002999250046875] |
|
2024-11-26 22:10:57,195 example INFO Train Epoch: 16 [90%] |
|
2024-11-26 22:10:57,195 example INFO [2.316251754760742, 3.02286696434021, 7.568589210510254, 17.739049911499023, 1.9846400022506714, 2.2880704402923584, 6600, 0.0002999250046875] |
|
2024-11-26 22:11:58,056 example INFO ====> Epoch: 16 |
|
2024-11-26 22:15:44,220 example INFO Train Epoch: 17 [39%] |
|
2024-11-26 22:15:44,220 example INFO [2.432720184326172, 2.6387364864349365, 7.3644537925720215, 17.754703521728516, 1.9778470993041992, 1.8624361753463745, 6800, 0.00029988751406191406] |
|
2024-11-26 22:20:35,615 example INFO Train Epoch: 17 [87%] |
|
2024-11-26 22:20:35,615 example INFO [2.361126661300659, 2.682238817214966, 8.34784984588623, 17.93964385986328, 2.0978760719299316, 2.3487472534179688, 7000, 0.00029988751406191406] |
|
2024-11-26 22:20:38,860 example INFO Saving model and optimizer state at iteration 17 to ./logs/example/G_7000.pth |
|
2024-11-26 22:20:39,742 example INFO Saving model and optimizer state at iteration 17 to ./logs/example/D_7000.pth |
|
2024-11-26 22:20:40,243 example INFO Saving model and optimizer state at iteration 17 to ./logs/example/DUR_7000.pth |
|
2024-11-26 22:20:40,345 example INFO .. Free up space by deleting ckpt ./logs/example/G_2000.pth |
|
2024-11-26 22:20:40,346 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_4000.pth |
|
2024-11-26 22:20:40,433 example INFO .. Free up space by deleting ckpt ./logs/example/D_5000.pth |
|
2024-11-26 22:21:55,328 example INFO ====> Epoch: 17 |
|
2024-11-26 22:25:16,059 example INFO Train Epoch: 18 [35%] |
|
2024-11-26 22:25:16,059 example INFO [2.284291982650757, 2.796096086502075, 9.167003631591797, 17.848011016845703, 2.114931583404541, 1.692778468132019, 7200, 0.00029985002812265633] |
|
2024-11-26 22:29:51,324 example INFO Train Epoch: 18 [83%] |
|
2024-11-26 22:29:51,325 example INFO [2.3119568824768066, 2.8867578506469727, 9.203574180603027, 18.501527786254883, 1.8910918235778809, 1.921372413635254, 7400, 0.00029985002812265633] |
|
2024-11-26 22:31:24,563 example INFO ====> Epoch: 18 |
|
2024-11-26 22:34:23,375 example INFO Train Epoch: 19 [31%] |
|
2024-11-26 22:34:23,376 example INFO [2.200601577758789, 2.6748135089874268, 9.464488983154297, 17.944517135620117, 2.0660295486450195, 1.9374104738235474, 7600, 0.000299812546869141] |
|
2024-11-26 22:39:12,689 example INFO Train Epoch: 19 [80%] |
|
2024-11-26 22:39:12,689 example INFO [2.3170197010040283, 2.3564746379852295, 8.148726463317871, 18.02412223815918, 2.0758750438690186, 2.1445038318634033, 7800, 0.000299812546869141] |
|
2024-11-26 22:41:05,365 example INFO ====> Epoch: 19 |
|
2024-11-26 22:43:42,718 example INFO Train Epoch: 20 [28%] |
|
2024-11-26 22:43:42,719 example INFO [2.1422860622406006, 2.6639230251312256, 9.424755096435547, 17.76228904724121, 1.8710048198699951, 2.7343218326568604, 8000, 0.00029977507030078235] |
|
2024-11-26 22:43:46,444 example INFO Saving model and optimizer state at iteration 20 to ./logs/example/G_8000.pth |
|
2024-11-26 22:43:47,333 example INFO Saving model and optimizer state at iteration 20 to ./logs/example/D_8000.pth |
|
2024-11-26 22:43:47,836 example INFO Saving model and optimizer state at iteration 20 to ./logs/example/DUR_8000.pth |
|
2024-11-26 22:43:47,938 example INFO .. Free up space by deleting ckpt ./logs/example/G_3000.pth |
|
2024-11-26 22:43:47,939 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_5000.pth |
|
2024-11-26 22:43:48,027 example INFO .. Free up space by deleting ckpt ./logs/example/D_6000.pth |
|
2024-11-26 22:48:17,112 example INFO Train Epoch: 20 [76%] |
|
2024-11-26 22:48:17,112 example INFO [2.3601958751678467, 2.73165225982666, 8.631917953491211, 17.476045608520508, 2.081468105316162, 1.9895514249801636, 8200, 0.00029977507030078235] |
|
2024-11-26 22:50:41,467 example INFO ====> Epoch: 20 |
|
2024-11-26 22:52:56,990 example INFO Train Epoch: 21 [24%] |
|
2024-11-26 22:52:56,990 example INFO [2.134584426879883, 2.666490316390991, 8.999197959899902, 18.431297302246094, 2.044191837310791, 2.190347194671631, 8400, 0.00029973759841699473] |
|
2024-11-26 22:57:36,222 example INFO Train Epoch: 21 [72%] |
|
2024-11-26 22:57:36,223 example INFO [2.5026357173919678, 2.555518627166748, 7.314276218414307, 17.213594436645508, 2.041895866394043, 1.9783430099487305, 8600, 0.00029973759841699473] |
|
2024-11-26 23:00:10,783 example INFO ====> Epoch: 21 |
|
2024-11-26 23:02:03,358 example INFO Train Epoch: 22 [20%] |
|
2024-11-26 23:02:03,359 example INFO [2.460747718811035, 2.3182170391082764, 8.613297462463379, 17.87166404724121, 1.9439833164215088, 1.851930856704712, 8800, 0.00029970013121719257] |
|
2024-11-26 23:06:31,632 example INFO Train Epoch: 22 [69%] |
|
2024-11-26 23:06:31,632 example INFO [2.334559917449951, 2.6329948902130127, 7.390477657318115, 16.69789695739746, 1.9627342224121094, 1.7777068614959717, 9000, 0.00029970013121719257] |
|
2024-11-26 23:06:34,965 example INFO Saving model and optimizer state at iteration 22 to ./logs/example/G_9000.pth |
|
2024-11-26 23:06:35,839 example INFO Saving model and optimizer state at iteration 22 to ./logs/example/D_9000.pth |
|
2024-11-26 23:06:36,342 example INFO Saving model and optimizer state at iteration 22 to ./logs/example/DUR_9000.pth |
|
2024-11-26 23:06:36,446 example INFO .. Free up space by deleting ckpt ./logs/example/G_4000.pth |
|
2024-11-26 23:06:36,447 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_6000.pth |
|
2024-11-26 23:06:36,537 example INFO .. Free up space by deleting ckpt ./logs/example/D_7000.pth |
|
2024-11-26 23:09:36,119 example INFO ====> Epoch: 22 |
|
2024-11-26 23:11:24,926 example INFO Train Epoch: 23 [17%] |
|
2024-11-26 23:11:24,926 example INFO [2.2470920085906982, 2.689209222793579, 8.270038604736328, 18.260164260864258, 1.9753196239471436, 2.4369730949401855, 9200, 0.0002996626687007904] |
|
2024-11-26 23:16:00,602 example INFO Train Epoch: 23 [65%] |
|
2024-11-26 23:16:00,603 example INFO [2.543705463409424, 2.451951265335083, 6.307395935058594, 16.25594139099121, 2.0077991485595703, 1.946689486503601, 9400, 0.0002996626687007904] |
|
2024-11-26 23:19:32,779 example INFO ====> Epoch: 23 |
|
2024-11-26 23:20:55,275 example INFO Train Epoch: 24 [13%] |
|
2024-11-26 23:20:55,275 example INFO [2.065840244293213, 2.8014044761657715, 8.693078994750977, 18.812402725219727, 1.9897873401641846, 1.7749162912368774, 9600, 0.0002996252108672028] |
|
2024-11-26 23:25:43,134 example INFO Train Epoch: 24 [61%] |
|
2024-11-26 23:25:43,135 example INFO [2.26464581489563, 2.851184844970703, 9.241716384887695, 18.037763595581055, 1.8824665546417236, 2.0746209621429443, 9800, 0.0002996252108672028] |
|
2024-11-26 23:29:33,098 example INFO ====> Epoch: 24 |
|
2024-11-26 23:30:32,463 example INFO Train Epoch: 25 [10%] |
|
2024-11-26 23:30:32,463 example INFO [2.5328032970428467, 2.561875104904175, 7.305269718170166, 17.647130966186523, 2.063962459564209, 1.9497959613800049, 10000, 0.00029958775771584436] |
|
2024-11-26 23:30:35,309 example INFO Saving model and optimizer state at iteration 25 to ./logs/example/G_10000.pth |
|
2024-11-26 23:30:36,134 example INFO Saving model and optimizer state at iteration 25 to ./logs/example/D_10000.pth |
|
2024-11-26 23:30:36,635 example INFO Saving model and optimizer state at iteration 25 to ./logs/example/DUR_10000.pth |
|
2024-11-26 23:30:36,736 example INFO .. Free up space by deleting ckpt ./logs/example/G_5000.pth |
|
2024-11-26 23:30:36,738 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_7000.pth |
|
2024-11-26 23:30:36,825 example INFO .. Free up space by deleting ckpt ./logs/example/D_8000.pth |
|
2024-11-26 23:35:08,821 example INFO Train Epoch: 25 [58%] |
|
2024-11-26 23:35:08,822 example INFO [2.498626232147217, 2.37683367729187, 7.875199317932129, 18.492572784423828, 1.9262491464614868, 1.785742998123169, 10200, 0.00029958775771584436] |
|
2024-11-26 23:39:01,052 example INFO ====> Epoch: 25 |
|
2024-11-26 23:39:37,525 example INFO Train Epoch: 26 [6%] |
|
2024-11-26 23:39:37,526 example INFO [2.437657356262207, 2.4753267765045166, 8.29281997680664, 18.43946075439453, 2.1178979873657227, 2.2250118255615234, 10400, 0.0002995503092461299] |
|
2024-11-26 23:44:17,103 example INFO Train Epoch: 26 [54%] |
|
2024-11-26 23:44:17,103 example INFO [2.4577488899230957, 2.4025685787200928, 6.876370429992676, 17.311969757080078, 1.9726569652557373, 1.839627742767334, 10600, 0.0002995503092461299] |
|
2024-11-26 23:48:34,921 example INFO ====> Epoch: 26 |
|
2024-11-26 23:48:52,247 example INFO Train Epoch: 27 [2%] |
|
2024-11-26 23:48:52,248 example INFO [2.1805777549743652, 2.547762870788574, 8.030771255493164, 17.838464736938477, 1.9239788055419922, 2.0353095531463623, 10800, 0.0002995128654574741] |
|
2024-11-26 23:53:28,541 example INFO Train Epoch: 27 [51%] |
|
2024-11-26 23:53:28,541 example INFO [2.363010883331299, 2.723341941833496, 7.701786041259766, 17.441492080688477, 1.9471265077590942, 2.072451591491699, 11000, 0.0002995128654574741] |
|
2024-11-26 23:53:31,189 example INFO Saving model and optimizer state at iteration 27 to ./logs/example/G_11000.pth |
|
2024-11-26 23:53:32,059 example INFO Saving model and optimizer state at iteration 27 to ./logs/example/D_11000.pth |
|
2024-11-26 23:53:32,555 example INFO Saving model and optimizer state at iteration 27 to ./logs/example/DUR_11000.pth |
|
2024-11-26 23:53:32,657 example INFO .. Free up space by deleting ckpt ./logs/example/G_6000.pth |
|
2024-11-26 23:53:32,658 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_8000.pth |
|
2024-11-26 23:53:32,746 example INFO .. Free up space by deleting ckpt ./logs/example/D_9000.pth |
|
2024-11-26 23:57:59,043 example INFO Train Epoch: 27 [99%] |
|
2024-11-26 23:57:59,043 example INFO [2.381420612335205, 2.4922637939453125, 7.7459821701049805, 18.202613830566406, 2.060837745666504, 1.878488302230835, 11200, 0.0002995128654574741] |
|
2024-11-26 23:58:04,615 example INFO ====> Epoch: 27 |
|
2024-11-27 00:02:25,565 example INFO Train Epoch: 28 [47%] |
|
2024-11-27 00:02:25,566 example INFO [2.4893126487731934, 2.4786462783813477, 6.269521713256836, 17.845630645751953, 2.2465767860412598, 2.4385335445404053, 11400, 0.0002994754263492919] |
|
2024-11-27 00:07:03,104 example INFO Train Epoch: 28 [95%] |
|
2024-11-27 00:07:03,104 example INFO [2.4674696922302246, 2.277906656265259, 6.878615379333496, 18.442781448364258, 2.0245742797851562, 1.85813307762146, 11600, 0.0002994754263492919] |
|
2024-11-27 00:07:28,709 example INFO ====> Epoch: 28 |
|
2024-11-27 00:11:31,321 example INFO Train Epoch: 29 [43%] |
|
2024-11-27 00:11:31,321 example INFO [2.3705859184265137, 2.7350246906280518, 8.191495895385742, 18.25433349609375, 2.0154013633728027, 2.1860413551330566, 11800, 0.00029943799192099825] |
|
2024-11-27 00:16:01,210 example INFO Train Epoch: 29 [92%] |
|
2024-11-27 00:16:01,211 example INFO [2.129507064819336, 2.971740245819092, 9.409879684448242, 18.365514755249023, 2.0683341026306152, 1.804132342338562, 12000, 0.00029943799192099825] |
|
2024-11-27 00:16:04,378 example INFO Saving model and optimizer state at iteration 29 to ./logs/example/G_12000.pth |
|
2024-11-27 00:16:05,262 example INFO Saving model and optimizer state at iteration 29 to ./logs/example/D_12000.pth |
|
2024-11-27 00:16:05,763 example INFO Saving model and optimizer state at iteration 29 to ./logs/example/DUR_12000.pth |
|
2024-11-27 00:16:05,867 example INFO .. Free up space by deleting ckpt ./logs/example/G_7000.pth |
|
2024-11-27 00:16:05,868 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_9000.pth |
|
2024-11-27 00:16:05,955 example INFO .. Free up space by deleting ckpt ./logs/example/D_10000.pth |
|
2024-11-27 00:16:49,913 example INFO ====> Epoch: 29 |
|
2024-11-27 00:20:58,797 example INFO Train Epoch: 30 [40%] |
|
2024-11-27 00:20:58,797 example INFO [1.802655816078186, 3.0505611896514893, 9.443607330322266, 17.753808975219727, 1.8996715545654297, 2.0129783153533936, 12200, 0.0002994005621720081] |
|
2024-11-27 00:25:32,969 example INFO Train Epoch: 30 [88%] |
|
2024-11-27 00:25:32,970 example INFO [2.5171234607696533, 2.5577821731567383, 8.124303817749023, 18.133832931518555, 1.7491014003753662, 1.8292235136032104, 12400, 0.0002994005621720081] |
|
2024-11-27 00:26:34,412 example INFO ====> Epoch: 30 |
|
2024-11-27 00:29:57,882 example INFO Train Epoch: 31 [36%] |
|
2024-11-27 00:29:57,882 example INFO [2.2517545223236084, 2.7667884826660156, 9.917475700378418, 19.357709884643555, 2.0009512901306152, 1.9003276824951172, 12600, 0.0002993631371017366] |
|
2024-11-27 00:34:27,315 example INFO Train Epoch: 31 [84%] |
|
2024-11-27 00:34:27,315 example INFO [2.3734261989593506, 2.401674747467041, 7.212299823760986, 17.84115219116211, 1.9962464570999146, 1.822561502456665, 12800, 0.0002993631371017366] |
|
2024-11-27 00:35:58,236 example INFO ====> Epoch: 31 |
|
2024-11-27 00:39:04,275 example INFO Train Epoch: 32 [33%] |
|
2024-11-27 00:39:04,275 example INFO [2.392998456954956, 2.5631253719329834, 9.138472557067871, 17.979612350463867, 1.9183712005615234, 2.138993501663208, 13000, 0.00029932571670959887] |
|
2024-11-27 00:39:07,232 example INFO Saving model and optimizer state at iteration 32 to ./logs/example/G_13000.pth |
|
2024-11-27 00:39:08,100 example INFO Saving model and optimizer state at iteration 32 to ./logs/example/D_13000.pth |
|
2024-11-27 00:39:08,595 example INFO Saving model and optimizer state at iteration 32 to ./logs/example/DUR_13000.pth |
|
2024-11-27 00:39:08,696 example INFO .. Free up space by deleting ckpt ./logs/example/G_8000.pth |
|
2024-11-27 00:39:08,698 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_10000.pth |
|
2024-11-27 00:39:08,785 example INFO .. Free up space by deleting ckpt ./logs/example/D_11000.pth |
|
2024-11-27 00:43:45,698 example INFO Train Epoch: 32 [81%] |
|
2024-11-27 00:43:45,699 example INFO [2.5083794593811035, 2.3832876682281494, 6.162660598754883, 16.94462776184082, 2.036337375640869, 2.171332359313965, 13200, 0.00029932571670959887] |
|
2024-11-27 00:45:31,496 example INFO ====> Epoch: 32 |
|
2024-11-27 00:48:16,852 example INFO Train Epoch: 33 [29%] |
|
2024-11-27 00:48:16,852 example INFO [1.9995133876800537, 2.9820306301116943, 9.220696449279785, 18.089176177978516, 1.9763833284378052, 1.9852354526519775, 13400, 0.00029928830099501015] |
|
2024-11-27 00:52:45,853 example INFO Train Epoch: 33 [77%] |
|
2024-11-27 00:52:45,853 example INFO [2.237306594848633, 2.6863059997558594, 8.595545768737793, 17.695674896240234, 2.0564982891082764, 1.959862470626831, 13600, 0.00029928830099501015] |
|
2024-11-27 00:54:47,978 example INFO ====> Epoch: 33 |
|
2024-11-27 00:57:22,318 example INFO Train Epoch: 34 [25%] |
|
2024-11-27 00:57:22,319 example INFO [2.166563034057617, 2.915870189666748, 9.000621795654297, 17.721942901611328, 1.9065220355987549, 2.2680251598358154, 13800, 0.00029925088995738576] |
|
2024-11-27 01:01:59,962 example INFO Train Epoch: 34 [73%] |
|
2024-11-27 01:01:59,963 example INFO [2.437307357788086, 2.469552993774414, 6.475400447845459, 16.828346252441406, 2.0508878231048584, 2.0867068767547607, 14000, 0.00029925088995738576] |
|
2024-11-27 01:02:02,761 example INFO Saving model and optimizer state at iteration 34 to ./logs/example/G_14000.pth |
|
2024-11-27 01:02:03,683 example INFO Saving model and optimizer state at iteration 34 to ./logs/example/D_14000.pth |
|
2024-11-27 01:02:04,226 example INFO Saving model and optimizer state at iteration 34 to ./logs/example/DUR_14000.pth |
|
2024-11-27 01:02:04,336 example INFO .. Free up space by deleting ckpt ./logs/example/G_9000.pth |
|
2024-11-27 01:02:04,337 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_11000.pth |
|
2024-11-27 01:02:04,432 example INFO .. Free up space by deleting ckpt ./logs/example/D_12000.pth |
|
2024-11-27 01:04:45,162 example INFO ====> Epoch: 34 |
|
2024-11-27 01:06:56,337 example INFO Train Epoch: 35 [22%] |
|
2024-11-27 01:06:56,338 example INFO [2.232225179672241, 2.6110568046569824, 7.822885036468506, 18.168445587158203, 2.0805017948150635, 1.7515413761138916, 14200, 0.0002992134835961411] |
|
2024-11-27 01:11:32,525 example INFO Train Epoch: 35 [70%] |
|
2024-11-27 01:11:32,525 example INFO [2.3253955841064453, 2.501091241836548, 7.584352493286133, 17.321107864379883, 1.9337491989135742, 1.9360854625701904, 14400, 0.0002992134835961411] |
|
2024-11-27 01:14:20,193 example INFO ====> Epoch: 35 |
|
2024-11-27 01:16:04,942 example INFO Train Epoch: 36 [18%] |
|
2024-11-27 01:16:04,942 example INFO [2.4838569164276123, 2.2130541801452637, 7.394598007202148, 18.28268051147461, 1.8792572021484375, 2.5089125633239746, 14600, 0.00029917608191069156] |
|
2024-11-27 01:20:28,463 example INFO Train Epoch: 36 [66%] |
|
2024-11-27 01:20:28,464 example INFO [2.5993692874908447, 2.2707881927490234, 7.906749725341797, 18.01835060119629, 2.1037330627441406, 2.0276124477386475, 14800, 0.00029917608191069156] |
|
2024-11-27 01:23:37,789 example INFO ====> Epoch: 36 |
|
2024-11-27 01:25:01,584 example INFO Train Epoch: 37 [14%] |
|
2024-11-27 01:25:01,584 example INFO [2.1746883392333984, 2.7950501441955566, 8.552321434020996, 17.983808517456055, 1.9931797981262207, 1.8981242179870605, 15000, 0.0002991386849004527] |
|
2024-11-27 01:25:04,597 example INFO Saving model and optimizer state at iteration 37 to ./logs/example/G_15000.pth |
|
2024-11-27 01:25:05,475 example INFO Saving model and optimizer state at iteration 37 to ./logs/example/D_15000.pth |
|
2024-11-27 01:25:05,975 example INFO Saving model and optimizer state at iteration 37 to ./logs/example/DUR_15000.pth |
|
2024-11-27 01:25:06,077 example INFO .. Free up space by deleting ckpt ./logs/example/G_10000.pth |
|
2024-11-27 01:25:06,078 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_12000.pth |
|
2024-11-27 01:25:06,165 example INFO .. Free up space by deleting ckpt ./logs/example/D_13000.pth |
|
2024-11-27 01:29:43,657 example INFO Train Epoch: 37 [63%] |
|
2024-11-27 01:29:43,657 example INFO [2.210141181945801, 3.012274742126465, 8.929434776306152, 17.762216567993164, 2.1699700355529785, 1.7823294401168823, 15200, 0.0002991386849004527] |
|
2024-11-27 01:33:15,077 example INFO ====> Epoch: 37 |
|
2024-11-27 01:34:19,720 example INFO Train Epoch: 38 [11%] |
|
2024-11-27 01:34:19,720 example INFO [2.3006296157836914, 2.4796247482299805, 8.51484203338623, 17.919729232788086, 1.885125994682312, 2.0947799682617188, 15400, 0.00029910129256484016] |
|
2024-11-27 01:38:50,171 example INFO Train Epoch: 38 [59%] |
|
2024-11-27 01:38:50,171 example INFO [2.142080307006836, 2.8825531005859375, 8.451852798461914, 18.043399810791016, 2.208256483078003, 2.0571510791778564, 15600, 0.00029910129256484016] |
|
2024-11-27 01:42:33,181 example INFO ====> Epoch: 38 |
|
2024-11-27 01:43:17,044 example INFO Train Epoch: 39 [7%] |
|
2024-11-27 01:43:17,045 example INFO [2.330936908721924, 2.6302425861358643, 7.44414758682251, 17.859617233276367, 2.0108413696289062, 1.9770585298538208, 15800, 0.0002990639049032695] |
|
2024-11-27 01:47:49,150 example INFO Train Epoch: 39 [55%] |
|
2024-11-27 01:47:49,150 example INFO [2.116626024246216, 2.757941246032715, 8.975098609924316, 17.47159194946289, 1.9811779260635376, 2.7741310596466064, 16000, 0.0002990639049032695] |
|
2024-11-27 01:47:51,984 example INFO Saving model and optimizer state at iteration 39 to ./logs/example/G_16000.pth |
|
2024-11-27 01:47:52,813 example INFO Saving model and optimizer state at iteration 39 to ./logs/example/D_16000.pth |
|
2024-11-27 01:47:53,312 example INFO Saving model and optimizer state at iteration 39 to ./logs/example/DUR_16000.pth |
|
2024-11-27 01:47:53,415 example INFO .. Free up space by deleting ckpt ./logs/example/G_11000.pth |
|
2024-11-27 01:47:53,416 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_13000.pth |
|
2024-11-27 01:47:53,503 example INFO .. Free up space by deleting ckpt ./logs/example/D_14000.pth |
|
2024-11-27 01:52:12,790 example INFO ====> Epoch: 39 |
|
2024-11-27 01:52:36,131 example INFO Train Epoch: 40 [4%] |
|
2024-11-27 01:52:36,132 example INFO [2.5386414527893066, 2.8039424419403076, 7.869595050811768, 17.906864166259766, 1.990480661392212, 2.0540099143981934, 16200, 0.0002990265219151566] |
|
2024-11-27 01:57:01,053 example INFO Train Epoch: 40 [52%] |
|
2024-11-27 01:57:01,054 example INFO [2.6139373779296875, 2.5915253162384033, 6.826314926147461, 17.62653350830078, 2.1228675842285156, 2.142118453979492, 16400, 0.0002990265219151566] |
|
2024-11-27 02:01:32,289 example INFO ====> Epoch: 40 |
|
2024-11-27 02:01:36,645 example INFO Train Epoch: 41 [0%] |
|
2024-11-27 02:01:36,645 example INFO [2.081435441970825, 2.770395517349243, 8.951592445373535, 18.770139694213867, 2.0880990028381348, 2.0428659915924072, 16600, 0.00029898914359991716] |
|
2024-11-27 02:05:55,975 example INFO Train Epoch: 41 [48%] |
|
2024-11-27 02:05:55,975 example INFO [2.380403518676758, 2.1720821857452393, 7.021974086761475, 17.30495834350586, 2.062861204147339, 2.2278804779052734, 16800, 0.00029898914359991716] |
|
2024-11-27 02:10:15,068 example INFO Train Epoch: 41 [96%] |
|
2024-11-27 02:10:15,069 example INFO [2.3521580696105957, 2.6648471355438232, 8.210667610168457, 16.941516876220703, 1.9228370189666748, 2.0681252479553223, 17000, 0.00029898914359991716] |
|
2024-11-27 02:10:17,450 example INFO Saving model and optimizer state at iteration 41 to ./logs/example/G_17000.pth |
|
2024-11-27 02:10:18,329 example INFO Saving model and optimizer state at iteration 41 to ./logs/example/D_17000.pth |
|
2024-11-27 02:10:18,821 example INFO Saving model and optimizer state at iteration 41 to ./logs/example/DUR_17000.pth |
|
2024-11-27 02:10:18,923 example INFO .. Free up space by deleting ckpt ./logs/example/G_12000.pth |
|
2024-11-27 02:10:18,924 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_14000.pth |
|
2024-11-27 02:10:19,012 example INFO .. Free up space by deleting ckpt ./logs/example/D_15000.pth |
|
2024-11-27 02:10:36,532 example INFO ====> Epoch: 41 |
|
2024-11-27 02:14:59,885 example INFO Train Epoch: 42 [45%] |
|
2024-11-27 02:14:59,886 example INFO [2.586900472640991, 2.448805093765259, 7.63529634475708, 17.155134201049805, 2.050759792327881, 1.8237131834030151, 17200, 0.00029895176995696715] |
|
2024-11-27 02:19:38,716 example INFO Train Epoch: 42 [93%] |
|
2024-11-27 02:19:38,716 example INFO [2.2233681678771973, 2.7241146564483643, 7.771903038024902, 17.97809410095215, 1.89956533908844, 2.2454965114593506, 17400, 0.00029895176995696715] |
|
2024-11-27 02:20:20,343 example INFO ====> Epoch: 42 |
|
2024-11-27 02:24:12,660 example INFO Train Epoch: 43 [41%] |
|
2024-11-27 02:24:12,660 example INFO [2.4052910804748535, 2.9449968338012695, 8.16640567779541, 18.738359451293945, 2.0187652111053467, 2.5720393657684326, 17600, 0.0002989144009857225] |
|
2024-11-27 02:28:51,067 example INFO Train Epoch: 43 [89%] |
|
2024-11-27 02:28:51,068 example INFO [2.1702558994293213, 2.7234575748443604, 8.338515281677246, 17.650135040283203, 1.8959381580352783, 2.131398916244507, 17800, 0.0002989144009857225] |
|
2024-11-27 02:29:49,253 example INFO ====> Epoch: 43 |
|
2024-11-27 02:33:33,061 example INFO Train Epoch: 44 [37%] |
|
2024-11-27 02:33:33,061 example INFO [2.5741405487060547, 2.820206642150879, 8.97107982635498, 17.87397575378418, 1.8477044105529785, 1.2544734477996826, 18000, 0.0002988770366855993] |
|
2024-11-27 02:33:35,832 example INFO Saving model and optimizer state at iteration 44 to ./logs/example/G_18000.pth |
|
2024-11-27 02:33:36,711 example INFO Saving model and optimizer state at iteration 44 to ./logs/example/D_18000.pth |
|
2024-11-27 02:33:37,215 example INFO Saving model and optimizer state at iteration 44 to ./logs/example/DUR_18000.pth |
|
2024-11-27 02:33:37,319 example INFO .. Free up space by deleting ckpt ./logs/example/G_13000.pth |
|
2024-11-27 02:33:37,320 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_15000.pth |
|
2024-11-27 02:33:37,408 example INFO .. Free up space by deleting ckpt ./logs/example/D_16000.pth |
|
2024-11-27 02:37:59,011 example INFO Train Epoch: 44 [86%] |
|
2024-11-27 02:37:59,011 example INFO [2.412214756011963, 2.4203786849975586, 8.405717849731445, 17.48372459411621, 2.074230909347534, 1.9170688390731812, 18200, 0.0002988770366855993] |
|
2024-11-27 02:39:22,111 example INFO ====> Epoch: 44 |
|
2024-11-27 02:42:24,427 example INFO Train Epoch: 45 [34%] |
|
2024-11-27 02:42:24,428 example INFO [2.3067846298217773, 2.8339734077453613, 8.432235717773438, 17.286026000976562, 2.048574924468994, 1.9226549863815308, 18400, 0.0002988396770560136] |
|
2024-11-27 02:46:51,585 example INFO Train Epoch: 45 [82%] |
|
2024-11-27 02:46:51,585 example INFO [2.2463631629943848, 2.8865480422973633, 8.490830421447754, 17.79404067993164, 2.0593607425689697, 1.6867209672927856, 18600, 0.0002988396770560136] |
|
2024-11-27 02:48:32,208 example INFO ====> Epoch: 45 |
|
2024-11-27 02:51:22,352 example INFO Train Epoch: 46 [30%] |
|
2024-11-27 02:51:22,353 example INFO [2.1531636714935303, 2.6901462078094482, 9.498796463012695, 18.17603302001953, 1.9899022579193115, 2.2338368892669678, 18800, 0.00029880232209638155] |
|
2024-11-27 02:55:47,432 example INFO Train Epoch: 46 [78%] |
|
2024-11-27 02:55:47,432 example INFO [2.314190626144409, 2.4729058742523193, 7.404723644256592, 17.820720672607422, 2.187959909439087, 1.972883939743042, 19000, 0.00029880232209638155] |
|
2024-11-27 02:55:50,043 example INFO Saving model and optimizer state at iteration 46 to ./logs/example/G_19000.pth |
|
2024-11-27 02:55:50,968 example INFO Saving model and optimizer state at iteration 46 to ./logs/example/D_19000.pth |
|
2024-11-27 02:55:51,510 example INFO Saving model and optimizer state at iteration 46 to ./logs/example/DUR_19000.pth |
|
2024-11-27 02:55:51,622 example INFO .. Free up space by deleting ckpt ./logs/example/G_14000.pth |
|
2024-11-27 02:55:51,623 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_16000.pth |
|
2024-11-27 02:55:51,718 example INFO .. Free up space by deleting ckpt ./logs/example/D_17000.pth |
|
2024-11-27 02:57:52,716 example INFO ====> Epoch: 46 |
|
2024-11-27 03:00:22,188 example INFO Train Epoch: 47 [27%] |
|
2024-11-27 03:00:22,189 example INFO [2.2215116024017334, 2.599796772003174, 7.838742256164551, 18.199687957763672, 2.0683212280273438, 2.1799349784851074, 19200, 0.0002987649718061195] |
|
2024-11-27 03:04:48,321 example INFO Train Epoch: 47 [75%] |
|
2024-11-27 03:04:48,322 example INFO [2.2541778087615967, 2.4042344093322754, 8.501060485839844, 17.696645736694336, 2.0932815074920654, 2.127061605453491, 19400, 0.0002987649718061195] |
|
2024-11-27 03:07:07,681 example INFO ====> Epoch: 47 |
|
2024-11-27 03:09:14,524 example INFO Train Epoch: 48 [23%] |
|
2024-11-27 03:09:14,524 example INFO [2.286815881729126, 2.677440881729126, 8.95881175994873, 18.017467498779297, 2.0652737617492676, 1.9983481168746948, 19600, 0.00029872762618464373] |
|
2024-11-27 03:13:44,922 example INFO Train Epoch: 48 [71%] |
|
2024-11-27 03:13:44,922 example INFO [2.27175235748291, 2.586759567260742, 7.945339202880859, 18.128034591674805, 1.951512336730957, 2.54514217376709, 19800, 0.00029872762618464373] |
|
2024-11-27 03:16:21,852 example INFO ====> Epoch: 48 |
|
2024-11-27 03:18:14,965 example INFO Train Epoch: 49 [19%] |
|
2024-11-27 03:18:14,966 example INFO [2.2748987674713135, 2.5766987800598145, 7.966551780700684, 18.70652961730957, 2.0426993370056152, 2.3794429302215576, 20000, 0.00029869028523137064] |
|
2024-11-27 03:18:17,689 example INFO Saving model and optimizer state at iteration 49 to ./logs/example/G_20000.pth |
|
2024-11-27 03:18:18,604 example INFO Saving model and optimizer state at iteration 49 to ./logs/example/D_20000.pth |
|
2024-11-27 03:18:19,148 example INFO Saving model and optimizer state at iteration 49 to ./logs/example/DUR_20000.pth |
|
2024-11-27 03:18:19,260 example INFO .. Free up space by deleting ckpt ./logs/example/G_15000.pth |
|
2024-11-27 03:18:19,261 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_17000.pth |
|
2024-11-27 03:18:19,356 example INFO .. Free up space by deleting ckpt ./logs/example/D_18000.pth |
|
2024-11-27 03:22:51,652 example INFO Train Epoch: 49 [67%] |
|
2024-11-27 03:22:51,652 example INFO [2.289407968521118, 2.863210439682007, 9.50819206237793, 17.76925277709961, 1.9643281698226929, 2.3349382877349854, 20200, 0.00029869028523137064] |
|
2024-11-27 03:25:45,194 example INFO ====> Epoch: 49 |
|
2024-11-27 03:27:13,632 example INFO Train Epoch: 50 [16%] |
|
2024-11-27 03:27:13,632 example INFO [2.244105577468872, 2.655681610107422, 9.269814491271973, 17.97189712524414, 1.7464172840118408, 1.7692911624908447, 20400, 0.0002986529489457167] |
|
2024-11-27 03:31:41,477 example INFO Train Epoch: 50 [64%] |
|
2024-11-27 03:31:41,477 example INFO [2.3338279724121094, 2.8875770568847656, 8.874505043029785, 17.61785316467285, 1.8605927228927612, 2.0407519340515137, 20600, 0.0002986529489457167] |
|
2024-11-27 03:34:56,485 example INFO ====> Epoch: 50 |
|
2024-11-27 03:36:08,126 example INFO Train Epoch: 51 [12%] |
|
2024-11-27 03:36:08,126 example INFO [2.421675682067871, 2.5295472145080566, 7.427030563354492, 18.402206420898438, 1.8898632526397705, 2.5297605991363525, 20800, 0.00029861561732709847] |
|
2024-11-27 03:40:34,616 example INFO Train Epoch: 51 [60%] |
|
2024-11-27 03:40:34,616 example INFO [2.4672951698303223, 2.834259033203125, 8.695383071899414, 17.778003692626953, 1.903224229812622, 1.9700735807418823, 21000, 0.00029861561732709847] |
|
2024-11-27 03:40:37,790 example INFO Saving model and optimizer state at iteration 51 to ./logs/example/G_21000.pth |
|
2024-11-27 03:40:38,655 example INFO Saving model and optimizer state at iteration 51 to ./logs/example/D_21000.pth |
|
2024-11-27 03:40:39,156 example INFO Saving model and optimizer state at iteration 51 to ./logs/example/DUR_21000.pth |
|
2024-11-27 03:40:39,258 example INFO .. Free up space by deleting ckpt ./logs/example/G_16000.pth |
|
2024-11-27 03:40:39,259 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_18000.pth |
|
2024-11-27 03:40:39,347 example INFO .. Free up space by deleting ckpt ./logs/example/D_19000.pth |
|
2024-11-27 03:44:17,178 example INFO ====> Epoch: 51 |
|
2024-11-27 03:45:11,337 example INFO Train Epoch: 52 [8%] |
|
2024-11-27 03:45:11,338 example INFO [2.230600357055664, 2.5793368816375732, 8.604133605957031, 17.148508071899414, 2.028534412384033, 1.9046767950057983, 21200, 0.00029857829037493257] |
|
2024-11-27 03:49:50,255 example INFO Train Epoch: 52 [57%] |
|
2024-11-27 03:49:50,256 example INFO [2.3205454349517822, 2.5727827548980713, 7.198722839355469, 17.05138397216797, 1.991906762123108, 2.0476396083831787, 21400, 0.00029857829037493257] |
|
2024-11-27 03:53:52,350 example INFO ====> Epoch: 52 |
|
2024-11-27 03:54:21,687 example INFO Train Epoch: 53 [5%] |
|
2024-11-27 03:54:21,688 example INFO [2.2376604080200195, 2.58538818359375, 8.815726280212402, 18.13877296447754, 1.9938442707061768, 1.9733145236968994, 21600, 0.0002985409680886357] |
|
2024-11-27 03:58:54,348 example INFO Train Epoch: 53 [53%] |
|
2024-11-27 03:58:54,348 example INFO [2.331653594970703, 2.881744861602783, 8.923091888427734, 17.536163330078125, 2.0328006744384766, 1.871918797492981, 21800, 0.0002985409680886357] |
|
2024-11-27 04:03:11,265 example INFO ====> Epoch: 53 |
|
2024-11-27 04:03:21,832 example INFO Train Epoch: 54 [1%] |
|
2024-11-27 04:03:21,832 example INFO [2.2777435779571533, 2.6691272258758545, 7.145444869995117, 17.034578323364258, 2.0429940223693848, 2.0569911003112793, 22000, 0.0002985036504676246] |
|
2024-11-27 04:03:24,640 example INFO Saving model and optimizer state at iteration 54 to ./logs/example/G_22000.pth |
|
2024-11-27 04:03:25,516 example INFO Saving model and optimizer state at iteration 54 to ./logs/example/D_22000.pth |
|
2024-11-27 04:03:26,016 example INFO Saving model and optimizer state at iteration 54 to ./logs/example/DUR_22000.pth |
|
2024-11-27 04:03:26,119 example INFO .. Free up space by deleting ckpt ./logs/example/G_17000.pth |
|
2024-11-27 04:03:26,120 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_19000.pth |
|
2024-11-27 04:03:26,209 example INFO .. Free up space by deleting ckpt ./logs/example/D_20000.pth |
|
2024-11-27 04:07:59,520 example INFO Train Epoch: 54 [49%] |
|
2024-11-27 04:07:59,520 example INFO [2.201810836791992, 2.6394481658935547, 9.311413764953613, 17.993398666381836, 1.8595901727676392, 1.9062360525131226, 22200, 0.0002985036504676246] |
|
2024-11-27 04:12:28,646 example INFO Train Epoch: 54 [98%] |
|
2024-11-27 04:12:28,646 example INFO [2.0802700519561768, 2.829209804534912, 8.48965072631836, 18.0351619720459, 1.9220385551452637, 1.683163046836853, 22400, 0.0002985036504676246] |
|
2024-11-27 04:12:40,749 example INFO ====> Epoch: 54 |
|
2024-11-27 04:16:59,578 example INFO Train Epoch: 55 [46%] |
|
2024-11-27 04:16:59,578 example INFO [2.123490810394287, 2.930896282196045, 9.419548034667969, 17.20577049255371, 2.0112130641937256, 2.053574800491333, 22600, 0.00029846633751131615] |
|
2024-11-27 04:21:29,871 example INFO Train Epoch: 55 [94%] |
|
2024-11-27 04:21:29,871 example INFO [2.1513590812683105, 2.54429292678833, 8.328696250915527, 18.453393936157227, 2.0693631172180176, 1.858181357383728, 22800, 0.00029846633751131615] |
|
2024-11-27 04:22:02,568 example INFO ====> Epoch: 55 |
|
2024-11-27 04:25:57,647 example INFO Train Epoch: 56 [42%] |
|
2024-11-27 04:25:57,647 example INFO [2.325803279876709, 2.938918113708496, 7.785640716552734, 17.86825180053711, 2.00283145904541, 1.6448652744293213, 23000, 0.00029842902921912724] |
|
2024-11-27 04:26:00,089 example INFO Saving model and optimizer state at iteration 56 to ./logs/example/G_23000.pth |
|
2024-11-27 04:26:00,915 example INFO Saving model and optimizer state at iteration 56 to ./logs/example/D_23000.pth |
|
2024-11-27 04:26:01,412 example INFO Saving model and optimizer state at iteration 56 to ./logs/example/DUR_23000.pth |
|
2024-11-27 04:26:01,514 example INFO .. Free up space by deleting ckpt ./logs/example/G_18000.pth |
|
2024-11-27 04:26:01,515 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_20000.pth |
|
2024-11-27 04:26:01,603 example INFO .. Free up space by deleting ckpt ./logs/example/D_21000.pth |
|
2024-11-27 04:30:38,469 example INFO Train Epoch: 56 [90%] |
|
2024-11-27 04:30:38,469 example INFO [2.3691349029541016, 2.593980312347412, 8.688641548156738, 16.912202835083008, 1.9229180812835693, 2.105417490005493, 23200, 0.00029842902921912724] |
|
2024-11-27 04:31:28,743 example INFO ====> Epoch: 56 |
|
2024-11-27 04:35:04,167 example INFO Train Epoch: 57 [39%] |
|
2024-11-27 04:35:04,167 example INFO [2.389885425567627, 2.6690826416015625, 8.347188949584961, 17.4355525970459, 1.9835230112075806, 2.1092262268066406, 23400, 0.0002983917255904748] |
|
2024-11-27 04:39:41,086 example INFO Train Epoch: 57 [87%] |
|
2024-11-27 04:39:41,087 example INFO [2.193607807159424, 3.087721109390259, 9.007079124450684, 17.63951301574707, 1.9491403102874756, 1.9337224960327148, 23600, 0.0002983917255904748] |
|
2024-11-27 04:40:51,963 example INFO ====> Epoch: 57 |
|
2024-11-27 04:44:05,942 example INFO Train Epoch: 58 [35%] |
|
2024-11-27 04:44:05,942 example INFO [2.140890121459961, 2.900073289871216, 8.083052635192871, 17.55701446533203, 2.075914144515991, 1.9804190397262573, 23800, 0.000298354426624776] |
|
2024-11-27 04:48:25,006 example INFO Train Epoch: 58 [83%] |
|
2024-11-27 04:48:25,007 example INFO [2.339219808578491, 2.3567121028900146, 7.391774654388428, 17.49967384338379, 2.1226487159729004, 1.9102517366409302, 24000, 0.000298354426624776] |
|
2024-11-27 04:48:27,710 example INFO Saving model and optimizer state at iteration 58 to ./logs/example/G_24000.pth |
|
2024-11-27 04:48:28,596 example INFO Saving model and optimizer state at iteration 58 to ./logs/example/D_24000.pth |
|
2024-11-27 04:48:29,097 example INFO Saving model and optimizer state at iteration 58 to ./logs/example/DUR_24000.pth |
|
2024-11-27 04:48:29,200 example INFO .. Free up space by deleting ckpt ./logs/example/G_19000.pth |
|
2024-11-27 04:48:29,201 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_21000.pth |
|
2024-11-27 04:48:29,288 example INFO .. Free up space by deleting ckpt ./logs/example/D_22000.pth |
|
2024-11-27 04:50:03,721 example INFO ====> Epoch: 58 |
|
2024-11-27 04:53:05,949 example INFO Train Epoch: 59 [31%] |
|
2024-11-27 04:53:05,950 example INFO [2.467021942138672, 2.5529375076293945, 8.558335304260254, 17.120773315429688, 1.9962376356124878, 1.9850317239761353, 24200, 0.0002983171323214479] |
|
2024-11-27 04:57:42,369 example INFO Train Epoch: 59 [80%] |
|
2024-11-27 04:57:42,369 example INFO [2.3398752212524414, 2.735565662384033, 9.103174209594727, 18.006996154785156, 1.9023362398147583, 2.137101650238037, 24400, 0.0002983171323214479] |
|
2024-11-27 04:59:34,506 example INFO ====> Epoch: 59 |
|
2024-11-27 05:02:21,809 example INFO Train Epoch: 60 [28%] |
|
2024-11-27 05:02:21,809 example INFO [2.1782314777374268, 2.5890018939971924, 9.579452514648438, 17.70368194580078, 2.043262004852295, 2.053614616394043, 24600, 0.0002982798426799077] |
|
2024-11-27 05:07:01,549 example INFO Train Epoch: 60 [76%] |
|
2024-11-27 05:07:01,550 example INFO [2.040818452835083, 2.9023993015289307, 9.251235961914062, 17.564035415649414, 1.9923627376556396, 2.379183530807495, 24800, 0.0002982798426799077] |
|
2024-11-27 05:09:14,998 example INFO ====> Epoch: 60 |
|
2024-11-27 05:11:33,467 example INFO Train Epoch: 61 [24%] |
|
2024-11-27 05:11:33,468 example INFO [2.322519063949585, 2.611250638961792, 7.473865032196045, 17.565982818603516, 1.9629125595092773, 1.8856505155563354, 25000, 0.0002982425576995727] |
|
2024-11-27 05:11:35,886 example INFO Saving model and optimizer state at iteration 61 to ./logs/example/G_25000.pth |
|
2024-11-27 05:11:36,765 example INFO Saving model and optimizer state at iteration 61 to ./logs/example/D_25000.pth |
|
2024-11-27 05:11:37,261 example INFO Saving model and optimizer state at iteration 61 to ./logs/example/DUR_25000.pth |
|
2024-11-27 05:11:37,363 example INFO .. Free up space by deleting ckpt ./logs/example/G_20000.pth |
|
2024-11-27 05:11:37,364 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_22000.pth |
|
2024-11-27 05:11:37,452 example INFO .. Free up space by deleting ckpt ./logs/example/D_23000.pth |
|
2024-11-27 05:16:06,328 example INFO Train Epoch: 61 [72%] |
|
2024-11-27 05:16:06,329 example INFO [2.2091658115386963, 2.7025718688964844, 8.394246101379395, 18.03936767578125, 1.9555672407150269, 2.195406913757324, 25200, 0.0002982425576995727] |
|
2024-11-27 05:18:39,705 example INFO ====> Epoch: 61 |
|
2024-11-27 05:20:38,813 example INFO Train Epoch: 62 [20%] |
|
2024-11-27 05:20:38,814 example INFO [2.6239051818847656, 2.463167190551758, 6.390732288360596, 18.3773193359375, 2.1289405822753906, 1.8884891271591187, 25400, 0.00029820527737986025] |
|
2024-11-27 05:25:16,875 example INFO Train Epoch: 62 [69%] |
|
2024-11-27 05:25:16,875 example INFO [2.6181893348693848, 2.2355782985687256, 6.851476192474365, 18.138227462768555, 2.186702251434326, 2.153459072113037, 25600, 0.00029820527737986025] |
|
2024-11-27 05:28:08,646 example INFO ====> Epoch: 62 |
|
2024-11-27 05:29:45,925 example INFO Train Epoch: 63 [17%] |
|
2024-11-27 05:29:45,926 example INFO [2.092756986618042, 3.0209145545959473, 9.828766822814941, 17.917743682861328, 1.7277370691299438, 2.091726541519165, 25800, 0.00029816800172018776] |
|
2024-11-27 05:34:12,838 example INFO Train Epoch: 63 [65%] |
|
2024-11-27 05:34:12,838 example INFO [2.4783401489257812, 2.2667205333709717, 8.628741264343262, 18.166950225830078, 2.175539016723633, 1.849973440170288, 26000, 0.00029816800172018776] |
|
2024-11-27 05:34:15,035 example INFO Saving model and optimizer state at iteration 63 to ./logs/example/G_26000.pth |
|
2024-11-27 05:34:16,435 example INFO Saving model and optimizer state at iteration 63 to ./logs/example/D_26000.pth |
|
2024-11-27 05:34:16,930 example INFO Saving model and optimizer state at iteration 63 to ./logs/example/DUR_26000.pth |
|
2024-11-27 05:34:17,032 example INFO .. Free up space by deleting ckpt ./logs/example/G_21000.pth |
|
2024-11-27 05:34:17,033 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_23000.pth |
|
2024-11-27 05:34:17,121 example INFO .. Free up space by deleting ckpt ./logs/example/D_24000.pth |
|
2024-11-27 05:37:33,992 example INFO ====> Epoch: 63 |
|
2024-11-27 05:38:50,418 example INFO Train Epoch: 64 [13%] |
|
2024-11-27 05:38:50,419 example INFO [2.1606881618499756, 2.744476556777954, 8.778894424438477, 18.022829055786133, 1.9250075817108154, 1.299869179725647, 26200, 0.0002981307307199727] |
|
2024-11-27 05:43:18,372 example INFO Train Epoch: 64 [61%] |
|
2024-11-27 05:43:18,372 example INFO [2.245103359222412, 2.7338690757751465, 8.376530647277832, 17.5057315826416, 2.032238006591797, 1.7516839504241943, 26400, 0.0002981307307199727] |
|
2024-11-27 05:46:49,667 example INFO ====> Epoch: 64 |
|
2024-11-27 05:47:50,552 example INFO Train Epoch: 65 [10%] |
|
2024-11-27 05:47:50,552 example INFO [2.531264305114746, 2.2742531299591064, 8.733835220336914, 17.854257583618164, 2.133175849914551, 2.272670030593872, 26600, 0.0002980934643786327] |
|
2024-11-27 05:52:17,219 example INFO Train Epoch: 65 [58%] |
|
2024-11-27 05:52:17,220 example INFO [2.4792914390563965, 2.6356654167175293, 7.736558437347412, 17.87808609008789, 2.0521793365478516, 2.149027109146118, 26800, 0.0002980934643786327] |
|
2024-11-27 05:56:10,269 example INFO ====> Epoch: 65 |
|
2024-11-27 05:56:47,987 example INFO Train Epoch: 66 [6%] |
|
2024-11-27 05:56:47,987 example INFO [2.0696730613708496, 2.960387945175171, 9.535909652709961, 17.71271324157715, 1.966367483139038, 2.0904593467712402, 27000, 0.00029805620269558535] |
|
2024-11-27 05:56:50,761 example INFO Saving model and optimizer state at iteration 66 to ./logs/example/G_27000.pth |
|
2024-11-27 05:56:51,640 example INFO Saving model and optimizer state at iteration 66 to ./logs/example/D_27000.pth |
|
2024-11-27 05:56:52,174 example INFO Saving model and optimizer state at iteration 66 to ./logs/example/DUR_27000.pth |
|
2024-11-27 05:56:52,284 example INFO .. Free up space by deleting ckpt ./logs/example/G_22000.pth |
|
2024-11-27 05:56:52,285 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_24000.pth |
|
2024-11-27 05:56:52,380 example INFO .. Free up space by deleting ckpt ./logs/example/D_25000.pth |
|
2024-11-27 06:01:12,215 example INFO Train Epoch: 66 [54%] |
|
2024-11-27 06:01:12,215 example INFO [2.3144547939300537, 2.7496652603149414, 9.35954761505127, 17.43228530883789, 2.0306055545806885, 1.9702574014663696, 27200, 0.00029805620269558535] |
|
2024-11-27 06:05:23,481 example INFO ====> Epoch: 66 |
|
2024-11-27 06:05:40,092 example INFO Train Epoch: 67 [2%] |
|
2024-11-27 06:05:40,092 example INFO [2.302874803543091, 2.6615827083587646, 9.004373550415039, 17.385948181152344, 1.8127431869506836, 1.7460614442825317, 27400, 0.00029801894567024837] |
|
2024-11-27 06:10:18,581 example INFO Train Epoch: 67 [51%] |
|
2024-11-27 06:10:18,581 example INFO [2.107560634613037, 2.7041404247283936, 8.751806259155273, 18.152603149414062, 1.9084439277648926, 2.2231285572052, 27600, 0.00029801894567024837] |
|
2024-11-27 06:14:55,211 example INFO Train Epoch: 67 [99%] |
|
2024-11-27 06:14:55,212 example INFO [2.462860107421875, 2.688795804977417, 8.45276165008545, 17.371490478515625, 1.9538600444793701, 2.7455687522888184, 27800, 0.00029801894567024837] |
|
2024-11-27 06:15:00,447 example INFO ====> Epoch: 67 |
|
2024-11-27 06:19:27,312 example INFO Train Epoch: 68 [47%] |
|
2024-11-27 06:19:27,312 example INFO [2.3437247276306152, 3.225512742996216, 9.745969772338867, 17.544719696044922, 2.162004232406616, 1.9489485025405884, 28000, 0.0002979816933020396] |
|
2024-11-27 06:19:29,711 example INFO Saving model and optimizer state at iteration 68 to ./logs/example/G_28000.pth |
|
2024-11-27 06:19:30,536 example INFO Saving model and optimizer state at iteration 68 to ./logs/example/D_28000.pth |
|
2024-11-27 06:19:31,035 example INFO Saving model and optimizer state at iteration 68 to ./logs/example/DUR_28000.pth |
|
2024-11-27 06:19:31,136 example INFO .. Free up space by deleting ckpt ./logs/example/G_23000.pth |
|
2024-11-27 06:19:31,138 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_25000.pth |
|
2024-11-27 06:19:31,226 example INFO .. Free up space by deleting ckpt ./logs/example/D_26000.pth |
|
2024-11-27 06:24:20,024 example INFO Train Epoch: 68 [95%] |
|
2024-11-27 06:24:20,024 example INFO [2.2074074745178223, 2.654315710067749, 9.322623252868652, 17.657873153686523, 1.985284447669983, 1.9657280445098877, 28200, 0.0002979816933020396] |
|
2024-11-27 06:24:48,818 example INFO ====> Epoch: 68 |
|
2024-11-27 06:29:08,000 example INFO Train Epoch: 69 [43%] |
|
2024-11-27 06:29:08,001 example INFO [2.2889487743377686, 2.558647632598877, 7.793576240539551, 17.620834350585938, 2.0256948471069336, 2.133063793182373, 28400, 0.00029794444559037684] |
|
2024-11-27 06:33:43,701 example INFO Train Epoch: 69 [92%] |
|
2024-11-27 06:33:43,701 example INFO [2.521854877471924, 2.4206109046936035, 6.703440189361572, 16.76975440979004, 2.125913381576538, 2.0332772731781006, 28600, 0.00029794444559037684] |
|
2024-11-27 06:34:29,888 example INFO ====> Epoch: 69 |
|
2024-11-27 06:38:26,279 example INFO Train Epoch: 70 [40%] |
|
2024-11-27 06:38:26,280 example INFO [2.3195903301239014, 2.371727466583252, 9.1450777053833, 17.73443603515625, 1.9274089336395264, 1.8788063526153564, 28800, 0.000297907202534678] |
|
2024-11-27 06:42:59,277 example INFO Train Epoch: 70 [88%] |
|
2024-11-27 06:42:59,278 example INFO [2.7118823528289795, 2.2644670009613037, 8.061635971069336, 17.467830657958984, 1.8710119724273682, 2.1718149185180664, 29000, 0.000297907202534678] |
|
2024-11-27 06:43:02,121 example INFO Saving model and optimizer state at iteration 70 to ./logs/example/G_29000.pth |
|
2024-11-27 06:43:02,996 example INFO Saving model and optimizer state at iteration 70 to ./logs/example/D_29000.pth |
|
2024-11-27 06:43:03,496 example INFO Saving model and optimizer state at iteration 70 to ./logs/example/DUR_29000.pth |
|
2024-11-27 06:43:03,598 example INFO .. Free up space by deleting ckpt ./logs/example/G_24000.pth |
|
2024-11-27 06:43:03,599 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_26000.pth |
|
2024-11-27 06:43:03,686 example INFO .. Free up space by deleting ckpt ./logs/example/D_27000.pth |
|
2024-11-27 06:44:07,131 example INFO ====> Epoch: 70 |
|
2024-11-27 06:47:35,101 example INFO Train Epoch: 71 [36%] |
|
2024-11-27 06:47:35,102 example INFO [2.29007887840271, 3.0707857608795166, 9.253173828125, 18.19451904296875, 1.98137366771698, 2.370471477508545, 29200, 0.0002978699641343612] |
|
2024-11-27 06:52:13,964 example INFO Train Epoch: 71 [84%] |
|
2024-11-27 06:52:13,964 example INFO [2.7040622234344482, 2.2951560020446777, 6.756763458251953, 17.669706344604492, 2.000687837600708, 2.0994069576263428, 29400, 0.0002978699641343612] |
|
2024-11-27 06:53:46,210 example INFO ====> Epoch: 71 |
|
2024-11-27 06:56:55,413 example INFO Train Epoch: 72 [33%] |
|
2024-11-27 06:56:55,414 example INFO [2.5213356018066406, 2.5595414638519287, 7.772063255310059, 17.71038246154785, 1.9245293140411377, 1.9553319215774536, 29600, 0.0002978327303888444] |
|
2024-11-27 07:01:23,407 example INFO Train Epoch: 72 [81%] |
|
2024-11-27 07:01:23,408 example INFO [2.101111888885498, 3.0711214542388916, 8.751062393188477, 18.608936309814453, 1.982896089553833, 2.1008167266845703, 29800, 0.0002978327303888444] |
|
2024-11-27 07:03:07,289 example INFO ====> Epoch: 72 |
|
2024-11-27 07:05:50,737 example INFO Train Epoch: 73 [29%] |
|
2024-11-27 07:05:50,737 example INFO [2.077723503112793, 2.8143296241760254, 8.920921325683594, 17.00605010986328, 1.9832953214645386, 1.920609712600708, 30000, 0.0002977955012975458] |
|
2024-11-27 07:05:54,303 example INFO Saving model and optimizer state at iteration 73 to ./logs/example/G_30000.pth |
|
2024-11-27 07:05:55,188 example INFO Saving model and optimizer state at iteration 73 to ./logs/example/D_30000.pth |
|
2024-11-27 07:05:55,692 example INFO Saving model and optimizer state at iteration 73 to ./logs/example/DUR_30000.pth |
|
2024-11-27 07:05:55,795 example INFO .. Free up space by deleting ckpt ./logs/example/G_25000.pth |
|
2024-11-27 07:05:55,796 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_27000.pth |
|
2024-11-27 07:05:55,883 example INFO .. Free up space by deleting ckpt ./logs/example/D_28000.pth |
|
2024-11-27 07:10:40,214 example INFO Train Epoch: 73 [77%] |
|
2024-11-27 07:10:40,215 example INFO [2.28376841545105, 2.9158623218536377, 9.32433032989502, 18.024076461791992, 1.8049637079238892, 1.686947226524353, 30200, 0.0002977955012975458] |
|
2024-11-27 07:12:50,937 example INFO ====> Epoch: 73 |
|
2024-11-27 07:15:16,229 example INFO Train Epoch: 74 [25%] |
|
2024-11-27 07:15:16,230 example INFO [2.3170268535614014, 2.3756401538848877, 6.9422149658203125, 17.021339416503906, 2.043532609939575, 1.8301644325256348, 30400, 0.0002977582768598836] |
|
2024-11-27 07:19:38,633 example INFO Train Epoch: 74 [73%] |
|
2024-11-27 07:19:38,634 example INFO [2.451871156692505, 2.5764248371124268, 8.106289863586426, 18.254125595092773, 1.9273831844329834, 2.0590057373046875, 30600, 0.0002977582768598836] |
|
2024-11-27 07:22:04,818 example INFO ====> Epoch: 74 |
|
2024-11-27 07:24:04,281 example INFO Train Epoch: 75 [22%] |
|
2024-11-27 07:24:04,281 example INFO [2.381500244140625, 2.775357723236084, 8.390664100646973, 17.347877502441406, 1.9790329933166504, 2.1395397186279297, 30800, 0.0002977210570752761] |
|
2024-11-27 07:28:24,599 example INFO Train Epoch: 75 [70%] |
|
2024-11-27 07:28:24,599 example INFO [2.044084310531616, 2.952972412109375, 10.0297212600708, 17.232582092285156, 1.9654752016067505, 1.8731341361999512, 31000, 0.0002977210570752761] |
|
2024-11-27 07:28:26,968 example INFO Saving model and optimizer state at iteration 75 to ./logs/example/G_31000.pth |
|
2024-11-27 07:28:27,845 example INFO Saving model and optimizer state at iteration 75 to ./logs/example/D_31000.pth |
|
2024-11-27 07:28:28,342 example INFO Saving model and optimizer state at iteration 75 to ./logs/example/DUR_31000.pth |
|
2024-11-27 07:28:28,445 example INFO .. Free up space by deleting ckpt ./logs/example/G_26000.pth |
|
2024-11-27 07:28:28,446 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_28000.pth |
|
2024-11-27 07:28:28,534 example INFO .. Free up space by deleting ckpt ./logs/example/D_29000.pth |
|
2024-11-27 07:31:23,103 example INFO ====> Epoch: 75 |
|
2024-11-27 07:33:04,365 example INFO Train Epoch: 76 [18%] |
|
2024-11-27 07:33:04,365 example INFO [2.1401755809783936, 2.8047616481781006, 10.258855819702148, 18.055557250976562, 1.8190277814865112, 1.5718215703964233, 31200, 0.0002976838419431417] |
|
2024-11-27 07:35:04,384 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 07:35:04,385 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 07:35:10,823 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 07:35:10,824 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 07:35:10,856 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 07:35:10,969 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 07:35:11,084 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 07:35:11,088 example INFO Loaded checkpoint './logs/example/DUR_31000.pth' (iteration 75) |
|
2024-11-27 07:35:11,457 example INFO Loaded checkpoint './logs/example/G_31000.pth' (iteration 75) |
|
2024-11-27 07:35:11,577 example INFO Loaded checkpoint './logs/example/D_31000.pth' (iteration 75) |
|
2024-11-27 07:46:08,370 example INFO Train Epoch: 75 [22%] |
|
2024-11-27 07:46:08,370 example INFO [2.4193625450134277, 2.6008434295654297, 8.323243141174316, 16.634788513183594, 1.9646912813186646, 1.8672147989273071, 30800, 0.0002999625] |
|
2024-11-27 07:59:51,975 example INFO Train Epoch: 75 [70%] |
|
2024-11-27 07:59:51,976 example INFO [2.255703926086426, 2.7642860412597656, 8.155168533325195, 17.446916580200195, 1.9971212148666382, 1.456869125366211, 31000, 0.0002999625] |
|
2024-11-27 07:59:56,471 example INFO Saving model and optimizer state at iteration 75 to ./logs/example/G_31000.pth |
|
2024-11-27 07:59:57,724 example INFO Saving model and optimizer state at iteration 75 to ./logs/example/D_31000.pth |
|
2024-11-27 07:59:58,510 example INFO Saving model and optimizer state at iteration 75 to ./logs/example/DUR_31000.pth |
|
2024-11-27 08:04:32,879 example INFO ====> Epoch: 75 |
|
2024-11-27 08:07:17,076 example INFO Train Epoch: 76 [18%] |
|
2024-11-27 08:07:17,077 example INFO [2.252368450164795, 2.8783504962921143, 8.667062759399414, 17.380096435546875, 1.834228754043579, 2.3242247104644775, 31200, 0.0002999250046875] |
|
2024-11-27 08:12:57,320 example INFO Train Epoch: 76 [66%] |
|
2024-11-27 08:12:57,321 example INFO [2.208488702774048, 2.8225436210632324, 8.6669340133667, 17.449485778808594, 1.9510983228683472, 1.9847686290740967, 31400, 0.0002999250046875] |
|
2024-11-27 08:16:38,515 example INFO ====> Epoch: 76 |
|
2024-11-27 08:18:06,505 example INFO Train Epoch: 77 [14%] |
|
2024-11-27 08:18:06,505 example INFO [2.1492128372192383, 2.7310023307800293, 8.02244758605957, 17.570661544799805, 1.9840505123138428, 1.8475395441055298, 31600, 0.00029988751406191406] |
|
2024-11-27 08:22:55,051 example INFO Train Epoch: 77 [63%] |
|
2024-11-27 08:22:55,051 example INFO [2.6804230213165283, 2.2663156986236572, 6.45174503326416, 15.990883827209473, 2.0268383026123047, 1.9131393432617188, 31800, 0.00029988751406191406] |
|
2024-11-27 08:26:36,058 example INFO ====> Epoch: 77 |
|
2024-11-27 08:27:46,750 example INFO Train Epoch: 78 [11%] |
|
2024-11-27 08:27:46,751 example INFO [2.2315099239349365, 2.7239768505096436, 8.016132354736328, 17.629257202148438, 2.0393481254577637, 1.5981884002685547, 32000, 0.00029985002812265633] |
|
2024-11-27 08:27:50,378 example INFO Saving model and optimizer state at iteration 78 to ./logs/example/G_32000.pth |
|
2024-11-27 08:27:51,331 example INFO Saving model and optimizer state at iteration 78 to ./logs/example/D_32000.pth |
|
2024-11-27 08:27:51,881 example INFO Saving model and optimizer state at iteration 78 to ./logs/example/DUR_32000.pth |
|
2024-11-27 08:27:51,992 example INFO .. Free up space by deleting ckpt ./logs/example/G_27000.pth |
|
2024-11-27 08:27:51,994 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_29000.pth |
|
2024-11-27 08:27:52,089 example INFO .. Free up space by deleting ckpt ./logs/example/D_30000.pth |
|
2024-11-27 08:32:43,713 example INFO Train Epoch: 78 [59%] |
|
2024-11-27 08:32:43,713 example INFO [2.296658754348755, 2.5734305381774902, 8.481379508972168, 18.04283332824707, 1.899171233177185, 1.8639733791351318, 32200, 0.00029985002812265633] |
|
2024-11-27 08:36:31,749 example INFO ====> Epoch: 78 |
|
2024-11-27 08:37:17,770 example INFO Train Epoch: 79 [7%] |
|
2024-11-27 08:37:17,771 example INFO [2.3770484924316406, 2.805541515350342, 7.974208354949951, 17.80265998840332, 2.1162209510803223, 1.8682494163513184, 32400, 0.000299812546869141] |
|
2024-11-27 08:41:58,229 example INFO Train Epoch: 79 [55%] |
|
2024-11-27 08:41:58,230 example INFO [1.923575520515442, 3.1750967502593994, 9.741873741149902, 19.065879821777344, 2.003067970275879, 2.387608289718628, 32600, 0.000299812546869141] |
|
2024-11-27 08:45:51,905 example INFO ====> Epoch: 79 |
|
2024-11-27 08:46:13,208 example INFO Train Epoch: 80 [4%] |
|
2024-11-27 08:46:13,209 example INFO [2.0507094860076904, 2.777179479598999, 8.020727157592773, 17.004009246826172, 2.0659866333007812, 2.2346267700195312, 32800, 0.00029977507030078235] |
|
2024-11-27 08:50:40,859 example INFO Train Epoch: 80 [52%] |
|
2024-11-27 08:50:40,859 example INFO [2.047499179840088, 2.8720459938049316, 10.33479118347168, 17.863441467285156, 2.109513282775879, 1.932076334953308, 33000, 0.00029977507030078235] |
|
2024-11-27 08:50:45,267 example INFO Saving model and optimizer state at iteration 80 to ./logs/example/G_33000.pth |
|
2024-11-27 08:50:46,213 example INFO Saving model and optimizer state at iteration 80 to ./logs/example/D_33000.pth |
|
2024-11-27 08:50:46,763 example INFO Saving model and optimizer state at iteration 80 to ./logs/example/DUR_33000.pth |
|
2024-11-27 08:50:46,876 example INFO .. Free up space by deleting ckpt ./logs/example/G_28000.pth |
|
2024-11-27 08:50:46,877 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_30000.pth |
|
2024-11-27 08:50:46,976 example INFO .. Free up space by deleting ckpt ./logs/example/D_31000.pth |
|
2024-11-27 08:55:20,341 example INFO ====> Epoch: 80 |
|
2024-11-27 08:55:22,934 example INFO Train Epoch: 81 [0%] |
|
2024-11-27 08:55:22,935 example INFO [2.44551420211792, 2.5209298133850098, 8.216099739074707, 17.431325912475586, 1.8782613277435303, 1.6197477579116821, 33200, 0.00029973759841699473] |
|
2024-11-27 09:00:03,687 example INFO Train Epoch: 81 [48%] |
|
2024-11-27 09:00:03,688 example INFO [2.711778402328491, 2.1855545043945312, 6.476617336273193, 17.342233657836914, 2.015225648880005, 1.9988614320755005, 33400, 0.00029973759841699473] |
|
2024-11-27 09:04:41,447 example INFO Train Epoch: 81 [96%] |
|
2024-11-27 09:04:41,448 example INFO [2.175603151321411, 2.6924378871917725, 9.029563903808594, 17.7795352935791, 1.9265305995941162, 1.6525673866271973, 33600, 0.00029973759841699473] |
|
2024-11-27 09:05:02,138 example INFO ====> Epoch: 81 |
|
2024-11-27 09:09:22,441 example INFO Train Epoch: 82 [45%] |
|
2024-11-27 09:09:22,442 example INFO [2.0372610092163086, 2.760497570037842, 9.393976211547852, 18.092449188232422, 2.0214083194732666, 1.869819164276123, 33800, 0.00029970013121719257] |
|
2024-11-27 09:14:09,496 example INFO Train Epoch: 82 [93%] |
|
2024-11-27 09:14:09,496 example INFO [2.08522629737854, 2.873894691467285, 10.165096282958984, 17.829280853271484, 1.9101884365081787, 2.1010348796844482, 34000, 0.00029970013121719257] |
|
2024-11-27 09:14:13,294 example INFO Saving model and optimizer state at iteration 82 to ./logs/example/G_34000.pth |
|
2024-11-27 09:14:14,228 example INFO Saving model and optimizer state at iteration 82 to ./logs/example/D_34000.pth |
|
2024-11-27 09:14:14,783 example INFO Saving model and optimizer state at iteration 82 to ./logs/example/DUR_34000.pth |
|
2024-11-27 09:14:14,894 example INFO .. Free up space by deleting ckpt ./logs/example/G_29000.pth |
|
2024-11-27 09:14:14,895 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_31000.pth |
|
2024-11-27 09:14:14,992 example INFO .. Free up space by deleting ckpt ./logs/example/D_32000.pth |
|
2024-11-27 09:15:01,597 example INFO ====> Epoch: 82 |
|
2024-11-27 09:18:56,584 example INFO Train Epoch: 83 [41%] |
|
2024-11-27 09:18:56,585 example INFO [1.816696047782898, 2.9738917350769043, 10.419486999511719, 17.890939712524414, 1.9989084005355835, 1.7473443746566772, 34200, 0.0002996626687007904] |
|
2024-11-27 09:23:33,138 example INFO Train Epoch: 83 [89%] |
|
2024-11-27 09:23:33,139 example INFO [2.167581796646118, 2.895143508911133, 8.705869674682617, 17.855079650878906, 1.8887526988983154, 1.939171552658081, 34400, 0.0002996626687007904] |
|
2024-11-27 09:24:30,043 example INFO ====> Epoch: 83 |
|
2024-11-27 09:28:16,743 example INFO Train Epoch: 84 [37%] |
|
2024-11-27 09:28:16,743 example INFO [2.666130781173706, 2.0095510482788086, 6.661293029785156, 16.904827117919922, 1.9460002183914185, 1.981949806213379, 34600, 0.0002996252108672028] |
|
2024-11-27 09:32:57,533 example INFO Train Epoch: 84 [86%] |
|
2024-11-27 09:32:57,533 example INFO [2.0408377647399902, 2.8783926963806152, 9.278785705566406, 17.481237411499023, 1.8950086832046509, 2.325122117996216, 34800, 0.0002996252108672028] |
|
2024-11-27 09:34:19,234 example INFO ====> Epoch: 84 |
|
2024-11-27 09:37:30,433 example INFO Train Epoch: 85 [34%] |
|
2024-11-27 09:37:30,433 example INFO [2.1877150535583496, 2.931749105453491, 9.450220108032227, 18.52570152282715, 1.837111473083496, 2.2657454013824463, 35000, 0.00029958775771584436] |
|
2024-11-27 09:37:34,375 example INFO Saving model and optimizer state at iteration 85 to ./logs/example/G_35000.pth |
|
2024-11-27 09:37:35,264 example INFO Saving model and optimizer state at iteration 85 to ./logs/example/D_35000.pth |
|
2024-11-27 09:37:35,809 example INFO Saving model and optimizer state at iteration 85 to ./logs/example/DUR_35000.pth |
|
2024-11-27 09:37:35,920 example INFO .. Free up space by deleting ckpt ./logs/example/G_30000.pth |
|
2024-11-27 09:37:35,922 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_32000.pth |
|
2024-11-27 09:37:36,019 example INFO .. Free up space by deleting ckpt ./logs/example/D_33000.pth |
|
2024-11-27 09:41:58,773 example INFO Train Epoch: 85 [82%] |
|
2024-11-27 09:41:58,774 example INFO [2.4476263523101807, 2.444704532623291, 8.647550582885742, 17.70772361755371, 1.923035740852356, 2.0695528984069824, 35200, 0.00029958775771584436] |
|
2024-11-27 09:43:37,191 example INFO ====> Epoch: 85 |
|
2024-11-27 09:46:26,464 example INFO Train Epoch: 86 [30%] |
|
2024-11-27 09:46:26,464 example INFO [2.6114208698272705, 2.5438759326934814, 7.828516006469727, 18.05434226989746, 2.0726661682128906, 1.9204730987548828, 35400, 0.0002995503092461299] |
|
2024-11-27 09:50:55,765 example INFO Train Epoch: 86 [78%] |
|
2024-11-27 09:50:55,766 example INFO [2.12890625, 2.5964818000793457, 8.224093437194824, 17.403818130493164, 1.9608476161956787, 1.801325798034668, 35600, 0.0002995503092461299] |
|
2024-11-27 09:52:46,266 example INFO ====> Epoch: 86 |
|
2024-11-27 09:55:19,119 example INFO Train Epoch: 87 [27%] |
|
2024-11-27 09:55:19,120 example INFO [2.5169732570648193, 2.296539783477783, 5.414093971252441, 16.886064529418945, 2.0945868492126465, 2.144853115081787, 35800, 0.0002995128654574741] |
|
2024-11-27 09:59:56,909 example INFO Train Epoch: 87 [75%] |
|
2024-11-27 09:59:56,909 example INFO [2.236753463745117, 2.4849488735198975, 8.5360107421875, 17.322084426879883, 1.8461198806762695, 1.9800903797149658, 36000, 0.0002995128654574741] |
|
2024-11-27 10:00:00,184 example INFO Saving model and optimizer state at iteration 87 to ./logs/example/G_36000.pth |
|
2024-11-27 10:00:01,077 example INFO Saving model and optimizer state at iteration 87 to ./logs/example/D_36000.pth |
|
2024-11-27 10:00:01,579 example INFO Saving model and optimizer state at iteration 87 to ./logs/example/DUR_36000.pth |
|
2024-11-27 10:00:01,682 example INFO .. Free up space by deleting ckpt ./logs/example/G_31000.pth |
|
2024-11-27 10:00:01,683 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_33000.pth |
|
2024-11-27 10:00:01,772 example INFO .. Free up space by deleting ckpt ./logs/example/D_34000.pth |
|
2024-11-27 10:02:20,428 example INFO ====> Epoch: 87 |
|
2024-11-27 10:04:29,295 example INFO Train Epoch: 88 [23%] |
|
2024-11-27 10:04:29,295 example INFO [2.204780340194702, 2.6042449474334717, 9.440164566040039, 17.499332427978516, 1.8596878051757812, 1.780368685722351, 36200, 0.0002994754263492919] |
|
2024-11-27 10:08:51,741 example INFO Train Epoch: 88 [71%] |
|
2024-11-27 10:08:51,741 example INFO [2.05206036567688, 2.7237963676452637, 8.592413902282715, 17.916101455688477, 1.9247514009475708, 2.123777389526367, 36400, 0.0002994754263492919] |
|
2024-11-27 10:11:29,608 example INFO ====> Epoch: 88 |
|
2024-11-27 10:13:21,426 example INFO Train Epoch: 89 [19%] |
|
2024-11-27 10:13:21,426 example INFO [2.2071638107299805, 2.6479833126068115, 9.297964096069336, 17.28972053527832, 1.9293293952941895, 2.0040292739868164, 36600, 0.00029943799192099825] |
|
2024-11-27 10:17:45,784 example INFO Train Epoch: 89 [67%] |
|
2024-11-27 10:17:45,784 example INFO [2.4242591857910156, 2.346362590789795, 8.088774681091309, 17.427146911621094, 1.8404779434204102, 1.5951714515686035, 36800, 0.00029943799192099825] |
|
2024-11-27 10:20:44,866 example INFO ====> Epoch: 89 |
|
2024-11-27 10:22:15,211 example INFO Train Epoch: 90 [16%] |
|
2024-11-27 10:22:15,211 example INFO [2.5128819942474365, 2.344923973083496, 8.267396926879883, 17.97853660583496, 2.0325164794921875, 2.186851739883423, 37000, 0.0002994005621720081] |
|
2024-11-27 10:22:18,668 example INFO Saving model and optimizer state at iteration 90 to ./logs/example/G_37000.pth |
|
2024-11-27 10:22:19,546 example INFO Saving model and optimizer state at iteration 90 to ./logs/example/D_37000.pth |
|
2024-11-27 10:22:20,041 example INFO Saving model and optimizer state at iteration 90 to ./logs/example/DUR_37000.pth |
|
2024-11-27 10:22:20,143 example INFO .. Free up space by deleting ckpt ./logs/example/G_32000.pth |
|
2024-11-27 10:22:20,145 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_34000.pth |
|
2024-11-27 10:22:20,233 example INFO .. Free up space by deleting ckpt ./logs/example/D_35000.pth |
|
2024-11-27 10:26:51,782 example INFO Train Epoch: 90 [64%] |
|
2024-11-27 10:26:51,782 example INFO [2.2291102409362793, 2.599703788757324, 7.948521614074707, 17.598974227905273, 1.9394488334655762, 2.383450984954834, 37200, 0.0002994005621720081] |
|
2024-11-27 10:30:21,452 example INFO ====> Epoch: 90 |
|
2024-11-27 10:31:33,128 example INFO Train Epoch: 91 [12%] |
|
2024-11-27 10:31:33,129 example INFO [2.0053610801696777, 2.8322832584381104, 9.548912048339844, 18.059246063232422, 1.8616852760314941, 1.9164701700210571, 37400, 0.0002993631371017366] |
|
2024-11-27 10:36:06,536 example INFO Train Epoch: 91 [60%] |
|
2024-11-27 10:36:06,537 example INFO [2.6096153259277344, 2.997239589691162, 9.651684761047363, 17.908283233642578, 2.003293514251709, 2.427511215209961, 37600, 0.0002993631371017366] |
|
2024-11-27 10:39:52,349 example INFO ====> Epoch: 91 |
|
2024-11-27 10:40:49,573 example INFO Train Epoch: 92 [8%] |
|
2024-11-27 10:40:49,573 example INFO [2.186727523803711, 2.752807855606079, 10.120664596557617, 17.447973251342773, 1.9396042823791504, 2.4073216915130615, 37800, 0.00029932571670959887] |
|
2024-11-27 10:45:11,457 example INFO Train Epoch: 92 [57%] |
|
2024-11-27 10:45:11,458 example INFO [2.3595783710479736, 2.513033151626587, 7.534233093261719, 16.58572769165039, 1.9588367938995361, 1.8545171022415161, 38000, 0.00029932571670959887] |
|
2024-11-27 10:45:14,772 example INFO Saving model and optimizer state at iteration 92 to ./logs/example/G_38000.pth |
|
2024-11-27 10:45:15,658 example INFO Saving model and optimizer state at iteration 92 to ./logs/example/D_38000.pth |
|
2024-11-27 10:45:16,176 example INFO Saving model and optimizer state at iteration 92 to ./logs/example/DUR_38000.pth |
|
2024-11-27 10:45:16,279 example INFO .. Free up space by deleting ckpt ./logs/example/G_33000.pth |
|
2024-11-27 10:45:16,280 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_35000.pth |
|
2024-11-27 10:45:16,370 example INFO .. Free up space by deleting ckpt ./logs/example/D_36000.pth |
|
2024-11-27 10:49:13,564 example INFO ====> Epoch: 92 |
|
2024-11-27 10:49:44,800 example INFO Train Epoch: 93 [5%] |
|
2024-11-27 10:49:44,801 example INFO [2.2280006408691406, 2.750096082687378, 8.659768104553223, 17.587963104248047, 2.0716874599456787, 2.1694517135620117, 38200, 0.00029928830099501015] |
|
2024-11-27 10:54:09,336 example INFO Train Epoch: 93 [53%] |
|
2024-11-27 10:54:09,336 example INFO [2.380234479904175, 2.4598774909973145, 8.808806419372559, 17.09217643737793, 1.9930481910705566, 2.1491236686706543, 38400, 0.00029928830099501015] |
|
2024-11-27 10:58:14,727 example INFO ====> Epoch: 93 |
|
2024-11-27 10:58:23,869 example INFO Train Epoch: 94 [1%] |
|
2024-11-27 10:58:23,869 example INFO [2.0375795364379883, 2.9967474937438965, 9.68545913696289, 17.900449752807617, 1.9905110597610474, 1.7296912670135498, 38600, 0.00029925088995738576] |
|
2024-11-27 11:02:49,562 example INFO Train Epoch: 94 [49%] |
|
2024-11-27 11:02:49,563 example INFO [2.6243600845336914, 2.111276626586914, 7.824564456939697, 17.409687042236328, 1.8538845777511597, 1.9172857999801636, 38800, 0.00029925088995738576] |
|
2024-11-27 11:07:20,837 example INFO Train Epoch: 94 [98%] |
|
2024-11-27 11:07:20,838 example INFO [2.5550193786621094, 2.1350739002227783, 8.205385208129883, 16.84824562072754, 1.8336150646209717, 2.147047996520996, 39000, 0.00029925088995738576] |
|
2024-11-27 11:07:24,410 example INFO Saving model and optimizer state at iteration 94 to ./logs/example/G_39000.pth |
|
2024-11-27 11:07:25,294 example INFO Saving model and optimizer state at iteration 94 to ./logs/example/D_39000.pth |
|
2024-11-27 11:07:25,818 example INFO Saving model and optimizer state at iteration 94 to ./logs/example/DUR_39000.pth |
|
2024-11-27 11:07:25,923 example INFO .. Free up space by deleting ckpt ./logs/example/G_34000.pth |
|
2024-11-27 11:07:25,925 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_36000.pth |
|
2024-11-27 11:07:26,014 example INFO .. Free up space by deleting ckpt ./logs/example/D_37000.pth |
|
2024-11-27 11:07:38,597 example INFO ====> Epoch: 94 |
|
2024-11-27 11:11:59,103 example INFO Train Epoch: 95 [46%] |
|
2024-11-27 11:11:59,104 example INFO [2.3140900135040283, 2.597670078277588, 7.50589656829834, 17.15851402282715, 2.0700533390045166, 2.324923276901245, 39200, 0.0002992134835961411] |
|
2024-11-27 11:16:25,235 example INFO Train Epoch: 95 [94%] |
|
2024-11-27 11:16:25,235 example INFO [2.0876872539520264, 2.876427412033081, 9.017943382263184, 17.18719482421875, 1.9940857887268066, 2.2040979862213135, 39400, 0.0002992134835961411] |
|
2024-11-27 11:16:56,574 example INFO ====> Epoch: 95 |
|
2024-11-27 11:20:51,558 example INFO Train Epoch: 96 [42%] |
|
2024-11-27 11:20:51,559 example INFO [2.4842610359191895, 2.314225912094116, 8.667492866516113, 17.3575439453125, 1.8036348819732666, 2.046863317489624, 39600, 0.00029917608191069156] |
|
2024-11-27 11:25:21,958 example INFO Train Epoch: 96 [90%] |
|
2024-11-27 11:25:21,959 example INFO [2.0075759887695312, 2.9660751819610596, 9.349228858947754, 18.164281845092773, 1.918877124786377, 2.28470516204834, 39800, 0.00029917608191069156] |
|
2024-11-27 11:26:13,693 example INFO ====> Epoch: 96 |
|
2024-11-27 11:29:49,074 example INFO Train Epoch: 97 [39%] |
|
2024-11-27 11:29:49,075 example INFO [2.177598237991333, 2.8372111320495605, 10.275385856628418, 17.636438369750977, 2.0033159255981445, 1.949869155883789, 40000, 0.0002991386849004527] |
|
2024-11-27 11:29:52,050 example INFO Saving model and optimizer state at iteration 97 to ./logs/example/G_40000.pth |
|
2024-11-27 11:29:52,978 example INFO Saving model and optimizer state at iteration 97 to ./logs/example/D_40000.pth |
|
2024-11-27 11:29:53,528 example INFO Saving model and optimizer state at iteration 97 to ./logs/example/DUR_40000.pth |
|
2024-11-27 11:29:53,639 example INFO .. Free up space by deleting ckpt ./logs/example/G_35000.pth |
|
2024-11-27 11:29:53,640 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_37000.pth |
|
2024-11-27 11:29:53,738 example INFO .. Free up space by deleting ckpt ./logs/example/D_38000.pth |
|
2024-11-27 11:34:22,058 example INFO Train Epoch: 97 [87%] |
|
2024-11-27 11:34:22,058 example INFO [2.3481369018554688, 2.64851975440979, 7.410446643829346, 17.72001075744629, 2.0500950813293457, 2.031402349472046, 40200, 0.0002991386849004527] |
|
2024-11-27 11:35:33,104 example INFO ====> Epoch: 97 |
|
2024-11-27 11:38:47,111 example INFO Train Epoch: 98 [35%] |
|
2024-11-27 11:38:47,112 example INFO [2.402674436569214, 2.588040828704834, 9.271367073059082, 17.232145309448242, 1.7837398052215576, 1.5371822118759155, 40400, 0.00029910129256484016] |
|
2024-11-27 11:43:15,347 example INFO Train Epoch: 98 [83%] |
|
2024-11-27 11:43:15,348 example INFO [2.361471652984619, 2.732083320617676, 7.938796520233154, 16.33742332458496, 1.8989850282669067, 2.185560941696167, 40600, 0.00029910129256484016] |
|
2024-11-27 11:44:41,982 example INFO ====> Epoch: 98 |
|
2024-11-27 11:47:32,146 example INFO Train Epoch: 99 [31%] |
|
2024-11-27 11:47:32,147 example INFO [2.338200569152832, 2.460628032684326, 7.657248497009277, 17.78666114807129, 1.9505938291549683, 2.1898083686828613, 40800, 0.0002990639049032695] |
|
2024-11-27 11:51:58,640 example INFO Train Epoch: 99 [80%] |
|
2024-11-27 11:51:58,641 example INFO [2.422417402267456, 2.60435152053833, 8.375060081481934, 17.506582260131836, 1.8624072074890137, 2.3337855339050293, 41000, 0.0002990639049032695] |
|
2024-11-27 11:52:01,411 example INFO Saving model and optimizer state at iteration 99 to ./logs/example/G_41000.pth |
|
2024-11-27 11:52:02,252 example INFO Saving model and optimizer state at iteration 99 to ./logs/example/D_41000.pth |
|
2024-11-27 11:52:02,750 example INFO Saving model and optimizer state at iteration 99 to ./logs/example/DUR_41000.pth |
|
2024-11-27 11:52:02,853 example INFO .. Free up space by deleting ckpt ./logs/example/G_36000.pth |
|
2024-11-27 11:52:02,855 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_38000.pth |
|
2024-11-27 11:52:02,943 example INFO .. Free up space by deleting ckpt ./logs/example/D_39000.pth |
|
2024-11-27 11:53:56,213 example INFO ====> Epoch: 99 |
|
2024-11-27 11:56:27,292 example INFO Train Epoch: 100 [28%] |
|
2024-11-27 11:56:27,292 example INFO [2.4120700359344482, 2.604050636291504, 8.065400123596191, 18.008617401123047, 2.091221332550049, 2.0792269706726074, 41200, 0.0002990265219151566] |
|
2024-11-27 12:00:52,635 example INFO Train Epoch: 100 [76%] |
|
2024-11-27 12:00:52,635 example INFO [2.3822779655456543, 2.5596060752868652, 9.156157493591309, 16.982389450073242, 1.8196442127227783, 1.946273684501648, 41400, 0.0002990265219151566] |
|
2024-11-27 12:03:07,569 example INFO ====> Epoch: 100 |
|
2024-11-27 12:05:27,269 example INFO Train Epoch: 101 [24%] |
|
2024-11-27 12:05:27,269 example INFO [2.5725996494293213, 2.268878936767578, 7.328193187713623, 17.942283630371094, 78.90674591064453, 2.845923662185669, 41600, 0.00029898914359991716] |
|
2024-11-27 12:09:48,964 example INFO Train Epoch: 101 [72%] |
|
2024-11-27 12:09:48,965 example INFO [2.2410945892333984, 2.903799533843994, 9.256604194641113, 18.163482666015625, 1.898111343383789, 2.2370212078094482, 41800, 0.00029898914359991716] |
|
2024-11-27 12:12:16,803 example INFO ====> Epoch: 101 |
|
2024-11-27 12:14:11,331 example INFO Train Epoch: 102 [20%] |
|
2024-11-27 12:14:11,331 example INFO [2.2081801891326904, 2.6960532665252686, 7.374100208282471, 17.676523208618164, 1.9901766777038574, 2.0929224491119385, 42000, 0.00029895176995696715] |
|
2024-11-27 12:14:14,312 example INFO Saving model and optimizer state at iteration 102 to ./logs/example/G_42000.pth |
|
2024-11-27 12:14:15,198 example INFO Saving model and optimizer state at iteration 102 to ./logs/example/D_42000.pth |
|
2024-11-27 12:14:15,707 example INFO Saving model and optimizer state at iteration 102 to ./logs/example/DUR_42000.pth |
|
2024-11-27 12:14:15,811 example INFO .. Free up space by deleting ckpt ./logs/example/G_37000.pth |
|
2024-11-27 12:14:15,812 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_39000.pth |
|
2024-11-27 12:14:15,900 example INFO .. Free up space by deleting ckpt ./logs/example/D_40000.pth |
|
2024-11-27 12:18:43,900 example INFO Train Epoch: 102 [69%] |
|
2024-11-27 12:18:43,900 example INFO [2.0327157974243164, 3.019935131072998, 9.752541542053223, 17.87929344177246, 1.792428731918335, 2.073279619216919, 42200, 0.00029895176995696715] |
|
2024-11-27 12:21:39,345 example INFO ====> Epoch: 102 |
|
2024-11-27 12:23:14,686 example INFO Train Epoch: 103 [17%] |
|
2024-11-27 12:23:14,687 example INFO [2.3308000564575195, 2.431323528289795, 8.202585220336914, 17.66350555419922, 2.0992679595947266, 1.9545460939407349, 42400, 0.0002989144009857225] |
|
2024-11-27 12:27:42,545 example INFO Train Epoch: 103 [65%] |
|
2024-11-27 12:27:42,545 example INFO [2.0117201805114746, 2.8904800415039062, 9.4350004196167, 17.395221710205078, 1.95753812789917, 2.1373274326324463, 42600, 0.0002989144009857225] |
|
2024-11-27 12:30:52,581 example INFO ====> Epoch: 103 |
|
2024-11-27 12:32:08,890 example INFO Train Epoch: 104 [13%] |
|
2024-11-27 12:32:08,890 example INFO [2.4376540184020996, 2.6796674728393555, 8.853902816772461, 17.355295181274414, 1.811611294746399, 1.7622160911560059, 42800, 0.0002988770366855993] |
|
2024-11-27 12:36:36,288 example INFO Train Epoch: 104 [61%] |
|
2024-11-27 12:36:36,289 example INFO [2.0837230682373047, 2.9659621715545654, 10.285778999328613, 17.740676879882812, 1.9696102142333984, 1.8081920146942139, 43000, 0.0002988770366855993] |
|
2024-11-27 12:36:39,879 example INFO Saving model and optimizer state at iteration 104 to ./logs/example/G_43000.pth |
|
2024-11-27 12:36:40,809 example INFO Saving model and optimizer state at iteration 104 to ./logs/example/D_43000.pth |
|
2024-11-27 12:36:41,353 example INFO Saving model and optimizer state at iteration 104 to ./logs/example/DUR_43000.pth |
|
2024-11-27 12:36:41,466 example INFO .. Free up space by deleting ckpt ./logs/example/G_38000.pth |
|
2024-11-27 12:36:41,467 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_40000.pth |
|
2024-11-27 12:36:41,563 example INFO .. Free up space by deleting ckpt ./logs/example/D_41000.pth |
|
2024-11-27 12:40:17,759 example INFO ====> Epoch: 104 |
|
2024-11-27 12:41:11,931 example INFO Train Epoch: 105 [10%] |
|
2024-11-27 12:41:11,931 example INFO [2.389725685119629, 2.6286227703094482, 8.482549667358398, 17.329517364501953, 1.8923567533493042, 1.5579028129577637, 43200, 0.0002988396770560136] |
|
2024-11-27 12:45:34,430 example INFO Train Epoch: 105 [58%] |
|
2024-11-27 12:45:34,431 example INFO [2.5344457626342773, 2.307063341140747, 6.658264636993408, 17.694921493530273, 2.0061850547790527, 1.6166973114013672, 43400, 0.0002988396770560136] |
|
2024-11-27 12:49:25,335 example INFO ====> Epoch: 105 |
|
2024-11-27 12:50:02,558 example INFO Train Epoch: 106 [6%] |
|
2024-11-27 12:50:02,558 example INFO [1.989676833152771, 2.804241180419922, 9.555879592895508, 17.498149871826172, 2.0064847469329834, 1.844304084777832, 43600, 0.00029880232209638155] |
|
2024-11-27 12:54:14,349 example INFO Train Epoch: 106 [54%] |
|
2024-11-27 12:54:14,349 example INFO [2.570152521133423, 2.0649518966674805, 7.158576011657715, 17.149127960205078, 1.9923137426376343, 1.3431142568588257, 43800, 0.00029880232209638155] |
|
2024-11-27 12:58:18,263 example INFO ====> Epoch: 106 |
|
2024-11-27 12:58:33,786 example INFO Train Epoch: 107 [2%] |
|
2024-11-27 12:58:33,786 example INFO [2.2702512741088867, 2.663048267364502, 8.49573802947998, 18.204452514648438, 1.9012893438339233, 2.3554980754852295, 44000, 0.0002987649718061195] |
|
2024-11-27 12:58:36,507 example INFO Saving model and optimizer state at iteration 107 to ./logs/example/G_44000.pth |
|
2024-11-27 12:58:37,436 example INFO Saving model and optimizer state at iteration 107 to ./logs/example/D_44000.pth |
|
2024-11-27 12:58:37,980 example INFO Saving model and optimizer state at iteration 107 to ./logs/example/DUR_44000.pth |
|
2024-11-27 12:58:38,089 example INFO .. Free up space by deleting ckpt ./logs/example/G_39000.pth |
|
2024-11-27 12:58:38,091 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_41000.pth |
|
2024-11-27 12:58:38,189 example INFO .. Free up space by deleting ckpt ./logs/example/D_42000.pth |
|
2024-11-27 13:02:59,818 example INFO Train Epoch: 107 [51%] |
|
2024-11-27 13:02:59,819 example INFO [2.6497554779052734, 2.5176377296447754, 8.363861083984375, 17.938087463378906, 1.935523271560669, 2.1533493995666504, 44200, 0.0002987649718061195] |
|
2024-11-27 13:07:29,441 example INFO Train Epoch: 107 [99%] |
|
2024-11-27 13:07:29,441 example INFO [1.9654412269592285, 3.298126459121704, 9.57739543914795, 17.4697208404541, 1.975977897644043, 1.5117169618606567, 44400, 0.0002987649718061195] |
|
2024-11-27 13:07:34,732 example INFO ====> Epoch: 107 |
|
2024-11-27 13:11:43,191 example INFO Train Epoch: 108 [47%] |
|
2024-11-27 13:11:43,192 example INFO [2.1885037422180176, 2.6210856437683105, 9.497023582458496, 18.38509750366211, 1.8911806344985962, 2.2897281646728516, 44600, 0.00029872762618464373] |
|
2024-11-27 13:16:08,749 example INFO Train Epoch: 108 [95%] |
|
2024-11-27 13:16:08,749 example INFO [2.4487297534942627, 2.4101340770721436, 7.24236536026001, 17.239646911621094, 1.9775371551513672, 1.582290768623352, 44800, 0.00029872762618464373] |
|
2024-11-27 13:16:33,100 example INFO ====> Epoch: 108 |
|
2024-11-27 13:20:33,084 example INFO Train Epoch: 109 [43%] |
|
2024-11-27 13:20:33,084 example INFO [2.9972662925720215, 1.7890938520431519, 6.3548903465271, 16.653213500976562, 1.7765530347824097, 1.7499440908432007, 45000, 0.00029869028523137064] |
|
2024-11-27 13:20:36,488 example INFO Saving model and optimizer state at iteration 109 to ./logs/example/G_45000.pth |
|
2024-11-27 13:20:37,390 example INFO Saving model and optimizer state at iteration 109 to ./logs/example/D_45000.pth |
|
2024-11-27 13:20:37,934 example INFO Saving model and optimizer state at iteration 109 to ./logs/example/DUR_45000.pth |
|
2024-11-27 13:20:38,046 example INFO .. Free up space by deleting ckpt ./logs/example/G_40000.pth |
|
2024-11-27 13:20:38,047 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_42000.pth |
|
2024-11-27 13:20:38,167 example INFO .. Free up space by deleting ckpt ./logs/example/D_43000.pth |
|
2024-11-27 13:24:56,769 example INFO Train Epoch: 109 [92%] |
|
2024-11-27 13:24:56,769 example INFO [2.5793163776397705, 2.4680354595184326, 9.254002571105957, 18.217529296875, 1.9154541492462158, 2.0608432292938232, 45200, 0.00029869028523137064] |
|
2024-11-27 13:25:44,607 example INFO ====> Epoch: 109 |
|
2024-11-27 13:29:42,501 example INFO Train Epoch: 110 [40%] |
|
2024-11-27 13:29:42,502 example INFO [2.2548491954803467, 2.4986820220947266, 7.429140090942383, 17.67620086669922, 1.9253336191177368, 1.7451248168945312, 45400, 0.0002986529489457167] |
|
2024-11-27 13:34:23,215 example INFO Train Epoch: 110 [88%] |
|
2024-11-27 13:34:23,216 example INFO [2.239521026611328, 2.59256911277771, 8.551736831665039, 17.47310447692871, 2.0121994018554688, 1.7969419956207275, 45600, 0.0002986529489457167] |
|
2024-11-27 13:35:28,844 example INFO ====> Epoch: 110 |
|
2024-11-27 13:38:57,487 example INFO Train Epoch: 111 [36%] |
|
2024-11-27 13:38:57,487 example INFO [2.303868293762207, 2.4074392318725586, 8.631715774536133, 17.275583267211914, 1.9121301174163818, 1.5280005931854248, 45800, 0.00029861561732709847] |
|
2024-11-27 13:43:21,100 example INFO Train Epoch: 111 [84%] |
|
2024-11-27 13:43:21,100 example INFO [2.12630558013916, 2.7620954513549805, 8.457351684570312, 18.19577407836914, 1.8170666694641113, 1.9316766262054443, 46000, 0.00029861561732709847] |
|
2024-11-27 13:43:23,661 example INFO Saving model and optimizer state at iteration 111 to ./logs/example/G_46000.pth |
|
2024-11-27 13:43:24,604 example INFO Saving model and optimizer state at iteration 111 to ./logs/example/D_46000.pth |
|
2024-11-27 13:43:25,150 example INFO Saving model and optimizer state at iteration 111 to ./logs/example/DUR_46000.pth |
|
2024-11-27 13:43:25,261 example INFO .. Free up space by deleting ckpt ./logs/example/G_41000.pth |
|
2024-11-27 13:43:25,262 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_43000.pth |
|
2024-11-27 13:43:25,359 example INFO .. Free up space by deleting ckpt ./logs/example/D_44000.pth |
|
2024-11-27 13:44:49,537 example INFO ====> Epoch: 111 |
|
2024-11-27 13:47:53,198 example INFO Train Epoch: 112 [33%] |
|
2024-11-27 13:47:53,199 example INFO [2.4815688133239746, 2.39723539352417, 8.027745246887207, 17.266231536865234, 1.9268991947174072, 2.0769712924957275, 46200, 0.00029857829037493257] |
|
2024-11-27 13:52:08,617 example INFO Train Epoch: 112 [81%] |
|
2024-11-27 13:52:08,618 example INFO [2.2979207038879395, 2.6720900535583496, 7.709920406341553, 17.7451171875, 1.8919401168823242, 2.201505184173584, 46400, 0.00029857829037493257] |
|
2024-11-27 13:54:05,402 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 13:54:05,403 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 13:54:12,012 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 13:54:12,012 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 13:54:12,047 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 13:54:12,162 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 13:54:12,296 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 13:54:12,299 example INFO Loaded checkpoint './logs/example/DUR_46000.pth' (iteration 111) |
|
2024-11-27 13:54:12,669 example INFO Loaded checkpoint './logs/example/G_46000.pth' (iteration 111) |
|
2024-11-27 13:54:12,806 example INFO Loaded checkpoint './logs/example/D_46000.pth' (iteration 111) |
|
2024-11-27 14:10:03,904 example INFO Train Epoch: 111 [36%] |
|
2024-11-27 14:10:03,905 example INFO [2.1274917125701904, 2.7735490798950195, 8.271595001220703, 18.323638916015625, 1.8894561529159546, 2.447422981262207, 45800, 0.0002999625] |
|
2024-11-27 14:20:10,153 example INFO Train Epoch: 111 [84%] |
|
2024-11-27 14:20:10,154 example INFO [2.380349636077881, 2.4350664615631104, 8.468555450439453, 17.34637451171875, 1.7997727394104004, 1.6279973983764648, 46000, 0.0002999625] |
|
2024-11-27 14:20:15,161 example INFO Saving model and optimizer state at iteration 111 to ./logs/example/G_46000.pth |
|
2024-11-27 14:20:16,367 example INFO Saving model and optimizer state at iteration 111 to ./logs/example/D_46000.pth |
|
2024-11-27 14:20:17,120 example INFO Saving model and optimizer state at iteration 111 to ./logs/example/DUR_46000.pth |
|
2024-11-27 14:22:35,873 example INFO ====> Epoch: 111 |
|
2024-11-27 14:27:15,194 example INFO Train Epoch: 112 [33%] |
|
2024-11-27 14:27:15,194 example INFO [2.045320987701416, 3.181579351425171, 9.420491218566895, 17.579025268554688, 2.0006818771362305, 2.1883134841918945, 46200, 0.0002999250046875] |
|
2024-11-27 14:32:54,093 example INFO Train Epoch: 112 [81%] |
|
2024-11-27 14:32:54,094 example INFO [2.245303153991699, 2.874271869659424, 9.138083457946777, 17.326419830322266, 1.9061319828033447, 2.1904592514038086, 46400, 0.0002999250046875] |
|
2024-11-27 14:34:53,302 example INFO ====> Epoch: 112 |
|
2024-11-27 14:38:05,764 example INFO Train Epoch: 113 [29%] |
|
2024-11-27 14:38:05,764 example INFO [2.364894151687622, 2.649961471557617, 8.477941513061523, 19.148359298706055, 2.0659427642822266, 1.948370099067688, 46600, 0.00029988751406191406] |
|
2024-11-27 14:43:12,373 example INFO Train Epoch: 113 [77%] |
|
2024-11-27 14:43:12,374 example INFO [2.3874971866607666, 2.587594985961914, 8.007402420043945, 17.803438186645508, 2.1614956855773926, 1.9662646055221558, 46800, 0.00029988751406191406] |
|
2024-11-27 14:45:25,539 example INFO ====> Epoch: 113 |
|
2024-11-27 14:47:54,970 example INFO Train Epoch: 114 [25%] |
|
2024-11-27 14:47:54,970 example INFO [2.4080395698547363, 2.21044921875, 6.045714855194092, 17.206682205200195, 1.9995172023773193, 1.8561606407165527, 47000, 0.00029985002812265633] |
|
2024-11-27 14:47:58,546 example INFO Saving model and optimizer state at iteration 114 to ./logs/example/G_47000.pth |
|
2024-11-27 14:47:59,433 example INFO Saving model and optimizer state at iteration 114 to ./logs/example/D_47000.pth |
|
2024-11-27 14:47:59,983 example INFO Saving model and optimizer state at iteration 114 to ./logs/example/DUR_47000.pth |
|
2024-11-27 14:48:00,098 example INFO .. Free up space by deleting ckpt ./logs/example/G_42000.pth |
|
2024-11-27 14:48:00,099 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_44000.pth |
|
2024-11-27 14:48:00,200 example INFO .. Free up space by deleting ckpt ./logs/example/D_45000.pth |
|
2024-11-27 14:52:54,963 example INFO Train Epoch: 114 [73%] |
|
2024-11-27 14:52:54,963 example INFO [2.398080587387085, 2.840363025665283, 8.436590194702148, 16.787038803100586, 1.811464786529541, 1.3223280906677246, 47200, 0.00029985002812265633] |
|
2024-11-27 14:55:28,481 example INFO ====> Epoch: 114 |
|
2024-11-27 14:57:46,387 example INFO Train Epoch: 115 [22%] |
|
2024-11-27 14:57:46,387 example INFO [2.2967042922973633, 2.6704392433166504, 8.712881088256836, 17.47995948791504, 1.8824869394302368, 2.011469841003418, 47400, 0.000299812546869141] |
|
2024-11-27 15:02:35,066 example INFO Train Epoch: 115 [70%] |
|
2024-11-27 15:02:35,067 example INFO [2.0146493911743164, 2.679407835006714, 9.624646186828613, 17.881227493286133, 1.9709891080856323, 1.8620712757110596, 47600, 0.000299812546869141] |
|
2024-11-27 15:05:31,589 example INFO ====> Epoch: 115 |
|
2024-11-27 15:07:15,717 example INFO Train Epoch: 116 [18%] |
|
2024-11-27 15:07:15,717 example INFO [2.1599090099334717, 2.854048252105713, 8.570719718933105, 18.060104370117188, 2.118725538253784, 1.9833415746688843, 47800, 0.00029977507030078235] |
|
2024-11-27 15:11:52,912 example INFO Train Epoch: 116 [66%] |
|
2024-11-27 15:11:52,913 example INFO [2.178083658218384, 2.7857589721679688, 9.237131118774414, 18.192045211791992, 1.8280770778656006, 1.3852720260620117, 48000, 0.00029977507030078235] |
|
2024-11-27 15:11:56,577 example INFO Saving model and optimizer state at iteration 116 to ./logs/example/G_48000.pth |
|
2024-11-27 15:11:57,455 example INFO Saving model and optimizer state at iteration 116 to ./logs/example/D_48000.pth |
|
2024-11-27 15:11:57,967 example INFO Saving model and optimizer state at iteration 116 to ./logs/example/DUR_48000.pth |
|
2024-11-27 15:11:58,071 example INFO .. Free up space by deleting ckpt ./logs/example/G_43000.pth |
|
2024-11-27 15:11:58,072 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_45000.pth |
|
2024-11-27 15:11:58,162 example INFO .. Free up space by deleting ckpt ./logs/example/D_46000.pth |
|
2024-11-27 15:15:21,390 example INFO ====> Epoch: 116 |
|
2024-11-27 15:16:54,743 example INFO Train Epoch: 117 [14%] |
|
2024-11-27 15:16:54,744 example INFO [1.9664885997772217, 3.077082633972168, 10.001106262207031, 17.772016525268555, 1.8868505954742432, 2.0291800498962402, 48200, 0.00029973759841699473] |
|
2024-11-27 15:22:59,941 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 15:22:59,941 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 15:23:06,362 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 15:23:06,363 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 15:23:06,396 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 15:23:06,508 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 15:23:06,548 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 15:23:06,551 example INFO Loaded checkpoint './logs/example/DUR_48000.pth' (iteration 116) |
|
2024-11-27 15:23:06,920 example INFO Loaded checkpoint './logs/example/G_48000.pth' (iteration 116) |
|
2024-11-27 15:23:07,057 example INFO Loaded checkpoint './logs/example/D_48000.pth' (iteration 116) |
|
2024-11-27 15:32:58,313 example INFO Train Epoch: 116 [18%] |
|
2024-11-27 15:32:58,313 example INFO [2.218064546585083, 2.455672025680542, 9.145730972290039, 18.059467315673828, 2.09875750541687, 2.1906747817993164, 47800, 0.0002999625] |
|
2024-11-27 15:45:57,376 example INFO Train Epoch: 116 [66%] |
|
2024-11-27 15:45:57,377 example INFO [2.335792064666748, 2.62579607963562, 8.184880256652832, 17.71673011779785, 1.7496219873428345, 1.5447752475738525, 48000, 0.0002999625] |
|
2024-11-27 15:46:01,808 example INFO Saving model and optimizer state at iteration 116 to ./logs/example/G_48000.pth |
|
2024-11-27 15:46:02,969 example INFO Saving model and optimizer state at iteration 116 to ./logs/example/D_48000.pth |
|
2024-11-27 15:46:03,764 example INFO Saving model and optimizer state at iteration 116 to ./logs/example/DUR_48000.pth |
|
2024-11-27 15:51:36,233 example INFO ====> Epoch: 116 |
|
2024-11-27 15:53:38,256 example INFO Train Epoch: 117 [14%] |
|
2024-11-27 15:53:38,257 example INFO [1.98058021068573, 3.1591145992279053, 10.287028312683105, 18.838708877563477, 1.893800139427185, 2.0518665313720703, 48200, 0.0002999250046875] |
|
2024-11-27 15:59:54,611 example INFO Train Epoch: 117 [63%] |
|
2024-11-27 15:59:54,611 example INFO [2.3359591960906982, 2.3814949989318848, 8.35322093963623, 17.451736450195312, 2.036752223968506, 1.7965879440307617, 48400, 0.0002999250046875] |
|
2024-11-27 16:03:52,920 example INFO ====> Epoch: 117 |
|
2024-11-27 16:04:55,668 example INFO Train Epoch: 118 [11%] |
|
2024-11-27 16:04:55,668 example INFO [2.126464366912842, 2.7261743545532227, 9.251364707946777, 17.657068252563477, 1.8032928705215454, 1.101110577583313, 48600, 0.00029988751406191406] |
|
2024-11-27 16:09:55,245 example INFO Train Epoch: 118 [59%] |
|
2024-11-27 16:09:55,246 example INFO [2.3756203651428223, 2.5647358894348145, 7.168493747711182, 17.05369758605957, 2.0100488662719727, 1.6808797121047974, 48800, 0.00029988751406191406] |
|
2024-11-27 16:13:53,134 example INFO ====> Epoch: 118 |
|
2024-11-27 16:14:38,877 example INFO Train Epoch: 119 [7%] |
|
2024-11-27 16:14:38,878 example INFO [2.4210081100463867, 2.5153591632843018, 6.315589904785156, 16.551502227783203, 1.9187860488891602, 1.6628901958465576, 49000, 0.00029985002812265633] |
|
2024-11-27 16:14:42,590 example INFO Saving model and optimizer state at iteration 119 to ./logs/example/G_49000.pth |
|
2024-11-27 16:14:43,443 example INFO Saving model and optimizer state at iteration 119 to ./logs/example/D_49000.pth |
|
2024-11-27 16:14:43,946 example INFO Saving model and optimizer state at iteration 119 to ./logs/example/DUR_49000.pth |
|
2024-11-27 16:14:44,048 example INFO .. Free up space by deleting ckpt ./logs/example/G_44000.pth |
|
2024-11-27 16:14:44,049 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_46000.pth |
|
2024-11-27 16:14:44,137 example INFO .. Free up space by deleting ckpt ./logs/example/D_47000.pth |
|
2024-11-27 16:19:23,606 example INFO Train Epoch: 119 [55%] |
|
2024-11-27 16:19:23,607 example INFO [2.277578830718994, 2.533203125, 9.157909393310547, 17.259502410888672, 1.8502801656723022, 1.8322721719741821, 49200, 0.00029985002812265633] |
|
2024-11-27 16:23:45,374 example INFO ====> Epoch: 119 |
|
2024-11-27 16:24:08,740 example INFO Train Epoch: 120 [4%] |
|
2024-11-27 16:24:08,740 example INFO [2.2287075519561768, 2.6168742179870605, 8.769237518310547, 17.338071823120117, 1.9328203201293945, 1.657271385192871, 49400, 0.000299812546869141] |
|
2024-11-27 16:28:32,153 example INFO Train Epoch: 120 [52%] |
|
2024-11-27 16:28:32,154 example INFO [2.135464906692505, 2.7761752605438232, 9.216083526611328, 17.34176254272461, 1.942373514175415, 2.2939274311065674, 49600, 0.000299812546869141] |
|
2024-11-27 16:33:01,017 example INFO ====> Epoch: 120 |
|
2024-11-27 16:33:04,488 example INFO Train Epoch: 121 [0%] |
|
2024-11-27 16:33:04,488 example INFO [2.3026819229125977, 2.724451780319214, 8.283604621887207, 18.18920135498047, 1.946961760520935, 1.649356722831726, 49800, 0.00029977507030078235] |
|
2024-11-27 16:37:33,029 example INFO Train Epoch: 121 [48%] |
|
2024-11-27 16:37:33,029 example INFO [2.056347608566284, 2.9795501232147217, 9.963462829589844, 18.120065689086914, 2.011791229248047, 1.8134099245071411, 50000, 0.00029977507030078235] |
|
2024-11-27 16:37:36,652 example INFO Saving model and optimizer state at iteration 121 to ./logs/example/G_50000.pth |
|
2024-11-27 16:37:38,464 example INFO Saving model and optimizer state at iteration 121 to ./logs/example/D_50000.pth |
|
2024-11-27 16:37:39,009 example INFO Saving model and optimizer state at iteration 121 to ./logs/example/DUR_50000.pth |
|
2024-11-27 16:37:39,120 example INFO .. Free up space by deleting ckpt ./logs/example/G_45000.pth |
|
2024-11-27 16:37:39,122 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_47000.pth |
|
2024-11-27 16:37:39,220 example INFO .. Free up space by deleting ckpt ./logs/example/D_48000.pth |
|
2024-11-27 16:42:13,326 example INFO Train Epoch: 121 [96%] |
|
2024-11-27 16:42:13,326 example INFO [2.269833564758301, 2.9322891235351562, 8.406538963317871, 18.139799118041992, 1.86897611618042, 1.812140941619873, 50200, 0.00029977507030078235] |
|
2024-11-27 16:42:31,327 example INFO ====> Epoch: 121 |
|
2024-11-27 16:46:44,033 example INFO Train Epoch: 122 [45%] |
|
2024-11-27 16:46:44,033 example INFO [2.2837722301483154, 2.612114667892456, 8.873598098754883, 17.98192024230957, 1.932052493095398, 1.8738527297973633, 50400, 0.00029973759841699473] |
|
2024-11-27 16:51:20,395 example INFO Train Epoch: 122 [93%] |
|
2024-11-27 16:51:20,395 example INFO [2.2946958541870117, 2.8418779373168945, 9.443933486938477, 17.028146743774414, 1.8742327690124512, 1.834377646446228, 50600, 0.00029973759841699473] |
|
2024-11-27 16:51:56,781 example INFO ====> Epoch: 122 |
|
2024-11-27 16:55:32,774 example INFO Train Epoch: 123 [41%] |
|
2024-11-27 16:55:32,774 example INFO [2.201589345932007, 2.5799388885498047, 9.725744247436523, 17.582277297973633, 1.9497706890106201, 1.3175561428070068, 50800, 0.00029970013121719257] |
|
2024-11-27 16:59:58,687 example INFO Train Epoch: 123 [89%] |
|
2024-11-27 16:59:58,688 example INFO [2.2585694789886475, 2.6399435997009277, 7.29685115814209, 18.28952980041504, 2.038606643676758, 2.0854058265686035, 51000, 0.00029970013121719257] |
|
2024-11-27 17:00:02,302 example INFO Saving model and optimizer state at iteration 123 to ./logs/example/G_51000.pth |
|
2024-11-27 17:00:03,248 example INFO Saving model and optimizer state at iteration 123 to ./logs/example/D_51000.pth |
|
2024-11-27 17:00:03,795 example INFO Saving model and optimizer state at iteration 123 to ./logs/example/DUR_51000.pth |
|
2024-11-27 17:00:03,909 example INFO .. Free up space by deleting ckpt ./logs/example/G_46000.pth |
|
2024-11-27 17:00:03,910 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_48000.pth |
|
2024-11-27 17:00:04,008 example INFO .. Free up space by deleting ckpt ./logs/example/D_49000.pth |
|
2024-11-27 17:01:09,329 example INFO ====> Epoch: 123 |
|
2024-11-27 17:04:36,474 example INFO Train Epoch: 124 [37%] |
|
2024-11-27 17:04:36,474 example INFO [2.1018948554992676, 3.290408134460449, 9.388442993164062, 18.461740493774414, 2.16048526763916, 2.212350606918335, 51200, 0.0002996626687007904] |
|
2024-11-27 17:08:50,324 example INFO Train Epoch: 124 [86%] |
|
2024-11-27 17:08:50,325 example INFO [2.0488948822021484, 3.430166244506836, 9.183769226074219, 17.832805633544922, 1.8018839359283447, 1.651784062385559, 51400, 0.0002996626687007904] |
|
2024-11-27 17:10:09,292 example INFO ====> Epoch: 124 |
|
2024-11-27 17:13:20,373 example INFO Train Epoch: 125 [34%] |
|
2024-11-27 17:13:20,373 example INFO [1.9519875049591064, 3.2492008209228516, 10.39100456237793, 17.5518856048584, 1.915605068206787, 1.7008583545684814, 51600, 0.0002996252108672028] |
|
2024-11-27 17:17:49,926 example INFO Train Epoch: 125 [82%] |
|
2024-11-27 17:17:49,927 example INFO [2.50352144241333, 2.414438009262085, 8.059969902038574, 16.399627685546875, 2.0753040313720703, 1.7051959037780762, 51800, 0.0002996252108672028] |
|
2024-11-27 17:19:23,334 example INFO ====> Epoch: 125 |
|
2024-11-27 17:22:26,665 example INFO Train Epoch: 126 [30%] |
|
2024-11-27 17:22:26,666 example INFO [2.2516722679138184, 2.9753615856170654, 8.377406120300293, 17.74110984802246, 1.9977686405181885, 1.8404695987701416, 52000, 0.00029958775771584436] |
|
2024-11-27 17:22:31,170 example INFO Saving model and optimizer state at iteration 126 to ./logs/example/G_52000.pth |
|
2024-11-27 17:22:32,070 example INFO Saving model and optimizer state at iteration 126 to ./logs/example/D_52000.pth |
|
2024-11-27 17:22:32,619 example INFO Saving model and optimizer state at iteration 126 to ./logs/example/DUR_52000.pth |
|
2024-11-27 17:22:32,731 example INFO .. Free up space by deleting ckpt ./logs/example/G_47000.pth |
|
2024-11-27 17:22:32,732 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_49000.pth |
|
2024-11-27 17:22:32,830 example INFO .. Free up space by deleting ckpt ./logs/example/D_50000.pth |
|
2024-11-27 17:27:12,813 example INFO Train Epoch: 126 [78%] |
|
2024-11-27 17:27:12,814 example INFO [1.9786128997802734, 3.149127244949341, 8.80140495300293, 17.364822387695312, 1.9528112411499023, 2.1450769901275635, 52200, 0.00029958775771584436] |
|
2024-11-27 17:29:14,193 example INFO ====> Epoch: 126 |
|
2024-11-27 17:31:48,108 example INFO Train Epoch: 127 [27%] |
|
2024-11-27 17:31:48,108 example INFO [2.427112340927124, 2.3431925773620605, 7.407600402832031, 18.39778709411621, 1.8895373344421387, 2.178515672683716, 52400, 0.0002995503092461299] |
|
2024-11-27 17:36:21,498 example INFO Train Epoch: 127 [75%] |
|
2024-11-27 17:36:21,499 example INFO [1.9598751068115234, 2.943223714828491, 9.791316032409668, 18.06472396850586, 1.9946017265319824, 0.562282919883728, 52600, 0.0002995503092461299] |
|
2024-11-27 17:38:49,425 example INFO ====> Epoch: 127 |
|
2024-11-27 17:41:02,458 example INFO Train Epoch: 128 [23%] |
|
2024-11-27 17:41:02,458 example INFO [2.3345186710357666, 2.4528286457061768, 9.296269416809082, 17.42926025390625, 1.9309165477752686, 2.2249038219451904, 52800, 0.0002995128654574741] |
|
2024-11-27 17:45:47,647 example INFO Train Epoch: 128 [71%] |
|
2024-11-27 17:45:47,648 example INFO [2.268549919128418, 2.8759303092956543, 8.968544006347656, 17.57685089111328, 1.9978435039520264, 2.0810649394989014, 53000, 0.0002995128654574741] |
|
2024-11-27 17:45:50,793 example INFO Saving model and optimizer state at iteration 128 to ./logs/example/G_53000.pth |
|
2024-11-27 17:45:51,688 example INFO Saving model and optimizer state at iteration 128 to ./logs/example/D_53000.pth |
|
2024-11-27 17:45:52,232 example INFO Saving model and optimizer state at iteration 128 to ./logs/example/DUR_53000.pth |
|
2024-11-27 17:45:52,344 example INFO .. Free up space by deleting ckpt ./logs/example/G_48000.pth |
|
2024-11-27 17:45:52,345 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_50000.pth |
|
2024-11-27 17:45:52,441 example INFO .. Free up space by deleting ckpt ./logs/example/D_51000.pth |
|
2024-11-27 17:48:35,942 example INFO ====> Epoch: 128 |
|
2024-11-27 17:50:31,703 example INFO Train Epoch: 129 [19%] |
|
2024-11-27 17:50:31,703 example INFO [2.223148822784424, 2.6993470191955566, 8.35459041595459, 17.048274993896484, 2.027312755584717, 1.9136898517608643, 53200, 0.0002994754263492919] |
|
2024-11-27 17:54:54,008 example INFO Train Epoch: 129 [67%] |
|
2024-11-27 17:54:54,009 example INFO [2.2702584266662598, 2.6037826538085938, 7.471068382263184, 18.49584197998047, 1.9592841863632202, 1.9746884107589722, 53400, 0.0002994754263492919] |
|
2024-11-27 17:57:54,065 example INFO ====> Epoch: 129 |
|
2024-11-27 17:59:25,524 example INFO Train Epoch: 130 [16%] |
|
2024-11-27 17:59:25,525 example INFO [2.269644021987915, 3.058723211288452, 8.798001289367676, 17.622610092163086, 1.8464179039001465, 2.251268148422241, 53600, 0.00029943799192099825] |
|
2024-11-27 18:04:06,014 example INFO Train Epoch: 130 [64%] |
|
2024-11-27 18:04:06,014 example INFO [2.027402400970459, 2.905505418777466, 9.333992004394531, 18.601974487304688, 2.031726837158203, 2.2994072437286377, 53800, 0.00029943799192099825] |
|
2024-11-27 18:07:29,376 example INFO ====> Epoch: 130 |
|
2024-11-27 18:08:35,745 example INFO Train Epoch: 131 [12%] |
|
2024-11-27 18:08:35,746 example INFO [1.9983713626861572, 2.8484396934509277, 9.166792869567871, 17.510066986083984, 1.890945553779602, 2.1942036151885986, 54000, 0.0002994005621720081] |
|
2024-11-27 18:08:39,370 example INFO Saving model and optimizer state at iteration 131 to ./logs/example/G_54000.pth |
|
2024-11-27 18:08:40,262 example INFO Saving model and optimizer state at iteration 131 to ./logs/example/D_54000.pth |
|
2024-11-27 18:08:40,810 example INFO Saving model and optimizer state at iteration 131 to ./logs/example/DUR_54000.pth |
|
2024-11-27 18:08:40,922 example INFO .. Free up space by deleting ckpt ./logs/example/G_49000.pth |
|
2024-11-27 18:08:40,923 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_51000.pth |
|
2024-11-27 18:08:41,018 example INFO .. Free up space by deleting ckpt ./logs/example/D_52000.pth |
|
2024-11-27 18:13:00,894 example INFO Train Epoch: 131 [60%] |
|
2024-11-27 18:13:00,894 example INFO [2.401341438293457, 2.366872787475586, 6.019070625305176, 16.116533279418945, 1.8962960243225098, 1.4212148189544678, 54200, 0.0002994005621720081] |
|
2024-11-27 18:16:44,237 example INFO ====> Epoch: 131 |
|
2024-11-27 18:17:33,195 example INFO Train Epoch: 132 [8%] |
|
2024-11-27 18:17:33,195 example INFO [2.244518280029297, 2.663771152496338, 7.937936305999756, 16.918413162231445, 1.95565927028656, 1.6027859449386597, 54400, 0.0002993631371017366] |
|
2024-11-27 18:22:18,047 example INFO Train Epoch: 132 [57%] |
|
2024-11-27 18:22:18,048 example INFO [2.5216431617736816, 2.746549367904663, 9.101592063903809, 17.81035614013672, 1.8395920991897583, 2.032074451446533, 54600, 0.0002993631371017366] |
|
2024-11-27 18:26:34,228 example INFO ====> Epoch: 132 |
|
2024-11-27 18:27:05,942 example INFO Train Epoch: 133 [5%] |
|
2024-11-27 18:27:05,943 example INFO [2.3363938331604004, 2.7230067253112793, 8.876866340637207, 16.87619972229004, 1.9225034713745117, 2.0526485443115234, 54800, 0.00029932571670959887] |
|
2024-11-27 18:31:37,942 example INFO Train Epoch: 133 [53%] |
|
2024-11-27 18:31:37,942 example INFO [2.341221332550049, 2.4980106353759766, 7.922154426574707, 17.487577438354492, 1.9167141914367676, 2.074425220489502, 55000, 0.00029932571670959887] |
|
2024-11-27 18:31:41,436 example INFO Saving model and optimizer state at iteration 133 to ./logs/example/G_55000.pth |
|
2024-11-27 18:31:42,278 example INFO Saving model and optimizer state at iteration 133 to ./logs/example/D_55000.pth |
|
2024-11-27 18:31:42,780 example INFO Saving model and optimizer state at iteration 133 to ./logs/example/DUR_55000.pth |
|
2024-11-27 18:31:42,882 example INFO .. Free up space by deleting ckpt ./logs/example/G_50000.pth |
|
2024-11-27 18:31:42,884 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_52000.pth |
|
2024-11-27 18:31:42,971 example INFO .. Free up space by deleting ckpt ./logs/example/D_53000.pth |
|
2024-11-27 18:35:56,079 example INFO ====> Epoch: 133 |
|
2024-11-27 18:36:06,533 example INFO Train Epoch: 134 [1%] |
|
2024-11-27 18:36:06,533 example INFO [2.2918202877044678, 2.5907387733459473, 7.303898811340332, 17.95252799987793, 1.9629309177398682, 2.054232120513916, 55200, 0.00029928830099501015] |
|
2024-11-27 18:40:42,266 example INFO Train Epoch: 134 [49%] |
|
2024-11-27 18:40:42,266 example INFO [2.2145915031433105, 2.692507266998291, 9.269808769226074, 17.64423942565918, 2.0520808696746826, 1.9651401042938232, 55400, 0.00029928830099501015] |
|
2024-11-27 18:45:16,260 example INFO Train Epoch: 134 [98%] |
|
2024-11-27 18:45:16,261 example INFO [2.0883750915527344, 2.749199151992798, 9.421571731567383, 18.257774353027344, 1.850992202758789, 1.8171052932739258, 55600, 0.00029928830099501015] |
|
2024-11-27 18:45:28,583 example INFO ====> Epoch: 134 |
|
2024-11-27 18:47:38,434 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 18:47:38,435 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 18:47:44,547 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 18:47:44,548 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 18:47:44,580 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 18:47:44,692 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 18:47:44,804 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 18:47:44,807 example INFO Loaded checkpoint './logs/example/DUR_55000.pth' (iteration 133) |
|
2024-11-27 18:47:45,161 example INFO Loaded checkpoint './logs/example/G_55000.pth' (iteration 133) |
|
2024-11-27 18:47:45,283 example INFO Loaded checkpoint './logs/example/D_55000.pth' (iteration 133) |
|
2024-11-27 18:51:23,359 example INFO Train Epoch: 133 [5%] |
|
2024-11-27 18:51:23,359 example INFO [2.567930221557617, 2.288343906402588, 7.278720855712891, 16.971715927124023, 1.9202929735183716, 1.95151948928833, 54800, 0.0002999625] |
|
2024-11-27 18:53:34,547 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 18:53:34,547 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 18:53:40,437 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 18:53:40,438 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 18:53:40,470 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 18:53:40,583 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 18:53:40,711 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 18:53:40,714 example INFO Loaded checkpoint './logs/example/DUR_55000.pth' (iteration 133) |
|
2024-11-27 18:53:41,078 example INFO Loaded checkpoint './logs/example/G_55000.pth' (iteration 133) |
|
2024-11-27 18:53:41,205 example INFO Loaded checkpoint './logs/example/D_55000.pth' (iteration 133) |
|
2024-11-27 18:57:21,646 example INFO Train Epoch: 133 [5%] |
|
2024-11-27 18:57:21,646 example INFO [2.505225896835327, 2.3645501136779785, 7.320204734802246, 16.887664794921875, 1.9267560243606567, 1.9369266033172607, 54800, 0.0002999625] |
|
2024-11-27 19:13:59,594 example INFO Train Epoch: 133 [53%] |
|
2024-11-27 19:13:59,595 example INFO [2.231886625289917, 2.8594467639923096, 8.587924003601074, 16.901317596435547, 1.911221981048584, 1.9167121648788452, 55000, 0.0002999625] |
|
2024-11-27 19:14:04,077 example INFO Saving model and optimizer state at iteration 133 to ./logs/example/G_55000.pth |
|
2024-11-27 19:14:05,262 example INFO Saving model and optimizer state at iteration 133 to ./logs/example/D_55000.pth |
|
2024-11-27 19:14:06,050 example INFO Saving model and optimizer state at iteration 133 to ./logs/example/DUR_55000.pth |
|
2024-11-27 19:22:45,855 example INFO ====> Epoch: 133 |
|
2024-11-27 19:23:02,990 example INFO Train Epoch: 134 [1%] |
|
2024-11-27 19:23:02,991 example INFO [2.596055030822754, 2.3204939365386963, 5.745488166809082, 17.168127059936523, 1.9542306661605835, 1.9484037160873413, 55200, 0.0002999250046875] |
|
2024-11-27 19:29:15,730 example INFO Train Epoch: 134 [49%] |
|
2024-11-27 19:29:15,731 example INFO [2.1295955181121826, 2.4481453895568848, 9.808348655700684, 17.643489837646484, 2.060991048812866, 2.4176836013793945, 55400, 0.0002999250046875] |
|
2024-11-27 19:34:43,102 example INFO Train Epoch: 134 [98%] |
|
2024-11-27 19:34:43,102 example INFO [2.297060966491699, 2.5300512313842773, 8.436284065246582, 18.69605255126953, 1.8575853109359741, 2.026651382446289, 55600, 0.0002999250046875] |
|
2024-11-27 19:34:57,239 example INFO ====> Epoch: 134 |
|
2024-11-27 19:39:51,094 example INFO Train Epoch: 135 [46%] |
|
2024-11-27 19:39:51,095 example INFO [2.2251651287078857, 2.8202755451202393, 8.914444923400879, 17.956745147705078, 1.93784761428833, 1.9246034622192383, 55800, 0.00029988751406191406] |
|
2024-11-27 19:44:47,080 example INFO Train Epoch: 135 [94%] |
|
2024-11-27 19:44:47,081 example INFO [2.1056504249572754, 3.286618709564209, 9.445638656616211, 17.447677612304688, 1.9775785207748413, 2.3815314769744873, 56000, 0.00029988751406191406] |
|
2024-11-27 19:44:50,738 example INFO Saving model and optimizer state at iteration 135 to ./logs/example/G_56000.pth |
|
2024-11-27 19:44:51,692 example INFO Saving model and optimizer state at iteration 135 to ./logs/example/D_56000.pth |
|
2024-11-27 19:44:52,244 example INFO Saving model and optimizer state at iteration 135 to ./logs/example/DUR_56000.pth |
|
2024-11-27 19:44:52,355 example INFO .. Free up space by deleting ckpt ./logs/example/G_51000.pth |
|
2024-11-27 19:44:52,357 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_53000.pth |
|
2024-11-27 19:44:52,456 example INFO .. Free up space by deleting ckpt ./logs/example/D_54000.pth |
|
2024-11-27 19:45:27,283 example INFO ====> Epoch: 135 |
|
2024-11-27 19:49:27,923 example INFO Train Epoch: 136 [42%] |
|
2024-11-27 19:49:27,923 example INFO [2.109224319458008, 3.151564359664917, 8.867328643798828, 17.686601638793945, 1.8470449447631836, 2.359971523284912, 56200, 0.00029985002812265633] |
|
2024-11-27 19:54:08,429 example INFO Train Epoch: 136 [90%] |
|
2024-11-27 19:54:08,430 example INFO [2.318875312805176, 2.2596728801727295, 6.297952651977539, 16.18889045715332, 1.9581354856491089, 2.288290023803711, 56400, 0.00029985002812265633] |
|
2024-11-27 19:55:03,252 example INFO ====> Epoch: 136 |
|
2024-11-27 19:58:45,039 example INFO Train Epoch: 137 [39%] |
|
2024-11-27 19:58:45,040 example INFO [2.4304275512695312, 2.708164930343628, 8.676051139831543, 17.573816299438477, 2.0001111030578613, 1.881616473197937, 56600, 0.000299812546869141] |
|
2024-11-27 20:03:11,527 example INFO Train Epoch: 137 [87%] |
|
2024-11-27 20:03:11,528 example INFO [2.319730758666992, 2.7166810035705566, 8.345190048217773, 17.30486488342285, 1.8609287738800049, 1.7593754529953003, 56800, 0.000299812546869141] |
|
2024-11-27 20:04:21,114 example INFO ====> Epoch: 137 |
|
2024-11-27 20:07:32,821 example INFO Train Epoch: 138 [35%] |
|
2024-11-27 20:07:32,821 example INFO [2.0033211708068848, 2.859894037246704, 8.631030082702637, 18.44962501525879, 1.9719343185424805, 1.9763919115066528, 57000, 0.00029977507030078235] |
|
2024-11-27 20:07:36,318 example INFO Saving model and optimizer state at iteration 138 to ./logs/example/G_57000.pth |
|
2024-11-27 20:07:37,217 example INFO Saving model and optimizer state at iteration 138 to ./logs/example/D_57000.pth |
|
2024-11-27 20:07:37,726 example INFO Saving model and optimizer state at iteration 138 to ./logs/example/DUR_57000.pth |
|
2024-11-27 20:07:37,829 example INFO .. Free up space by deleting ckpt ./logs/example/G_52000.pth |
|
2024-11-27 20:07:37,831 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_54000.pth |
|
2024-11-27 20:07:37,920 example INFO .. Free up space by deleting ckpt ./logs/example/D_55000.pth |
|
2024-11-27 20:12:11,921 example INFO Train Epoch: 138 [83%] |
|
2024-11-27 20:12:11,922 example INFO [2.182281494140625, 2.958920955657959, 8.4590482711792, 17.25461769104004, 1.962517261505127, 1.8944356441497803, 57200, 0.00029977507030078235] |
|
2024-11-27 20:13:45,137 example INFO ====> Epoch: 138 |
|
2024-11-27 20:16:47,839 example INFO Train Epoch: 139 [31%] |
|
2024-11-27 20:16:47,839 example INFO [2.6370058059692383, 2.2119832038879395, 7.741240501403809, 16.502531051635742, 1.8353441953659058, 1.8393478393554688, 57400, 0.00029973759841699473] |
|
2024-11-27 20:21:15,306 example INFO Train Epoch: 139 [80%] |
|
2024-11-27 20:21:15,306 example INFO [1.8914775848388672, 3.0131478309631348, 10.048016548156738, 16.938411712646484, 1.91901433467865, 1.8406994342803955, 57600, 0.00029973759841699473] |
|
2024-11-27 20:23:21,319 example INFO ====> Epoch: 139 |
|
2024-11-27 20:26:07,557 example INFO Train Epoch: 140 [28%] |
|
2024-11-27 20:26:07,558 example INFO [1.8844740390777588, 2.9135398864746094, 9.52561092376709, 16.77011489868164, 2.184486150741577, 2.3911733627319336, 57800, 0.00029970013121719257] |
|
2024-11-27 20:30:38,765 example INFO Train Epoch: 140 [76%] |
|
2024-11-27 20:30:38,766 example INFO [2.243731737136841, 2.6357388496398926, 8.620634078979492, 17.596769332885742, 1.8532607555389404, 1.9189127683639526, 58000, 0.00029970013121719257] |
|
2024-11-27 20:30:42,219 example INFO Saving model and optimizer state at iteration 140 to ./logs/example/G_58000.pth |
|
2024-11-27 20:30:43,105 example INFO Saving model and optimizer state at iteration 140 to ./logs/example/D_58000.pth |
|
2024-11-27 20:30:43,653 example INFO Saving model and optimizer state at iteration 140 to ./logs/example/DUR_58000.pth |
|
2024-11-27 20:30:43,764 example INFO .. Free up space by deleting ckpt ./logs/example/G_53000.pth |
|
2024-11-27 20:30:43,765 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_55000.pth |
|
2024-11-27 20:30:43,861 example INFO .. Free up space by deleting ckpt ./logs/example/D_56000.pth |
|
2024-11-27 20:33:05,270 example INFO ====> Epoch: 140 |
|
2024-11-27 20:34:19,640 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 20:34:19,641 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 20:34:25,081 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 20:34:25,082 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 20:34:25,136 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 20:34:25,273 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 20:34:25,312 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 20:34:25,315 example INFO Loaded checkpoint './logs/example/DUR_58000.pth' (iteration 140) |
|
2024-11-27 20:34:25,719 example INFO Loaded checkpoint './logs/example/G_58000.pth' (iteration 140) |
|
2024-11-27 20:34:25,840 example INFO Loaded checkpoint './logs/example/D_58000.pth' (iteration 140) |
|
2024-11-27 20:48:03,149 example INFO Train Epoch: 140 [28%] |
|
2024-11-27 20:48:03,150 example INFO [2.4282031059265137, 2.3888838291168213, 7.959798812866211, 16.897968292236328, 1.957763433456421, 2.159717082977295, 57800, 0.0002999625] |
|
2024-11-27 20:59:35,134 example INFO Train Epoch: 140 [76%] |
|
2024-11-27 20:59:35,134 example INFO [2.307203769683838, 2.5059726238250732, 9.127669334411621, 17.974512100219727, 1.9025464057922363, 2.0534913539886475, 58000, 0.0002999625] |
|
2024-11-27 20:59:39,488 example INFO Saving model and optimizer state at iteration 140 to ./logs/example/G_58000.pth |
|
2024-11-27 20:59:41,182 example INFO Saving model and optimizer state at iteration 140 to ./logs/example/D_58000.pth |
|
2024-11-27 20:59:41,926 example INFO Saving model and optimizer state at iteration 140 to ./logs/example/DUR_58000.pth |
|
2024-11-27 21:03:42,615 example INFO ====> Epoch: 140 |
|
2024-11-27 21:07:01,461 example INFO Train Epoch: 141 [24%] |
|
2024-11-27 21:07:01,462 example INFO [2.4672906398773193, 2.7891898155212402, 7.561331272125244, 17.688953399658203, 1.955309271812439, 2.0227770805358887, 58200, 0.0002999250046875] |
|
2024-11-27 21:12:19,659 example INFO Train Epoch: 141 [72%] |
|
2024-11-27 21:12:19,660 example INFO [2.2463088035583496, 2.5422539710998535, 8.804997444152832, 17.216196060180664, 1.911784291267395, 2.283026933670044, 58400, 0.0002999250046875] |
|
2024-11-27 21:15:10,664 example INFO ====> Epoch: 141 |
|
2024-11-27 21:17:15,301 example INFO Train Epoch: 142 [20%] |
|
2024-11-27 21:17:15,302 example INFO [2.4844954013824463, 2.681180000305176, 8.443764686584473, 18.435592651367188, 2.005460500717163, 1.8732166290283203, 58600, 0.00029988751406191406] |
|
2024-11-27 21:22:07,098 example INFO Train Epoch: 142 [69%] |
|
2024-11-27 21:22:07,099 example INFO [2.4931864738464355, 2.346268892288208, 7.606515407562256, 16.15656280517578, 1.9334911108016968, 1.7751169204711914, 58800, 0.00029988751406191406] |
|
2024-11-27 21:25:13,387 example INFO ====> Epoch: 142 |
|
2024-11-27 21:26:56,632 example INFO Train Epoch: 143 [17%] |
|
2024-11-27 21:26:56,633 example INFO [2.2919695377349854, 3.08060884475708, 8.922222137451172, 17.59687042236328, 2.0086288452148438, 2.0415122509002686, 59000, 0.00029985002812265633] |
|
2024-11-27 21:27:00,023 example INFO Saving model and optimizer state at iteration 143 to ./logs/example/G_59000.pth |
|
2024-11-27 21:27:00,973 example INFO Saving model and optimizer state at iteration 143 to ./logs/example/D_59000.pth |
|
2024-11-27 21:27:01,520 example INFO Saving model and optimizer state at iteration 143 to ./logs/example/DUR_59000.pth |
|
2024-11-27 21:27:01,632 example INFO .. Free up space by deleting ckpt ./logs/example/G_54000.pth |
|
2024-11-27 21:27:01,633 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_56000.pth |
|
2024-11-27 21:27:01,730 example INFO .. Free up space by deleting ckpt ./logs/example/D_57000.pth |
|
2024-11-27 21:31:42,389 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 21:31:42,390 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 21:31:48,372 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 21:31:48,373 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 21:31:48,422 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 21:31:48,557 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 21:31:48,602 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 21:31:48,607 example INFO Loaded checkpoint './logs/example/DUR_59000.pth' (iteration 143) |
|
2024-11-27 21:31:49,115 example INFO Loaded checkpoint './logs/example/G_59000.pth' (iteration 143) |
|
2024-11-27 21:31:49,273 example INFO Loaded checkpoint './logs/example/D_59000.pth' (iteration 143) |
|
2024-11-27 21:41:46,034 example INFO Train Epoch: 143 [17%] |
|
2024-11-27 21:41:46,034 example INFO [2.3962771892547607, 2.659025192260742, 8.236784934997559, 18.01763153076172, 1.9661123752593994, 2.0093955993652344, 59000, 0.0002999625] |
|
2024-11-27 21:41:52,469 example INFO Saving model and optimizer state at iteration 143 to ./logs/example/G_59000.pth |
|
2024-11-27 21:41:53,657 example INFO Saving model and optimizer state at iteration 143 to ./logs/example/D_59000.pth |
|
2024-11-27 21:41:54,410 example INFO Saving model and optimizer state at iteration 143 to ./logs/example/DUR_59000.pth |
|
2024-11-27 21:55:21,466 example INFO Train Epoch: 143 [65%] |
|
2024-11-27 21:55:21,467 example INFO [2.1334760189056396, 2.9792168140411377, 9.139891624450684, 18.450754165649414, 1.9185214042663574, 1.6325095891952515, 59200, 0.0002999625] |
|
2024-11-27 22:00:52,516 example INFO ====> Epoch: 143 |
|
2024-11-27 22:02:42,881 example INFO Train Epoch: 144 [13%] |
|
2024-11-27 22:02:42,881 example INFO [2.1986918449401855, 2.976162910461426, 9.208578109741211, 16.92201805114746, 1.8547643423080444, 1.1069716215133667, 59400, 0.0002999250046875] |
|
2024-11-27 22:08:45,647 example INFO Train Epoch: 144 [61%] |
|
2024-11-27 22:08:45,647 example INFO [2.368537425994873, 2.485743284225464, 7.292294025421143, 17.047983169555664, 1.9722857475280762, 1.8864706754684448, 59600, 0.0002999250046875] |
|
2024-11-27 22:13:07,623 example INFO ====> Epoch: 144 |
|
2024-11-27 22:14:10,586 example INFO Train Epoch: 145 [10%] |
|
2024-11-27 22:14:10,586 example INFO [2.321446180343628, 2.567497968673706, 9.025667190551758, 17.225933074951172, 1.8771252632141113, 2.147679567337036, 59800, 0.00029988751406191406] |
|
2024-11-27 22:19:06,178 example INFO Train Epoch: 145 [58%] |
|
2024-11-27 22:19:06,178 example INFO [2.1713337898254395, 2.764096260070801, 8.915590286254883, 16.904680252075195, 2.0788869857788086, 1.805922508239746, 60000, 0.00029988751406191406] |
|
2024-11-27 22:19:09,468 example INFO Saving model and optimizer state at iteration 145 to ./logs/example/G_60000.pth |
|
2024-11-27 22:19:10,865 example INFO Saving model and optimizer state at iteration 145 to ./logs/example/D_60000.pth |
|
2024-11-27 22:19:11,366 example INFO Saving model and optimizer state at iteration 145 to ./logs/example/DUR_60000.pth |
|
2024-11-27 22:19:11,470 example INFO .. Free up space by deleting ckpt ./logs/example/G_55000.pth |
|
2024-11-27 22:19:11,471 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_57000.pth |
|
2024-11-27 22:19:11,560 example INFO .. Free up space by deleting ckpt ./logs/example/D_58000.pth |
|
2024-11-27 22:23:08,075 example INFO ====> Epoch: 145 |
|
2024-11-27 22:23:48,861 example INFO Train Epoch: 146 [6%] |
|
2024-11-27 22:23:48,861 example INFO [2.3552350997924805, 2.6219942569732666, 7.1495232582092285, 18.395153045654297, 1.9652354717254639, 2.0755443572998047, 60200, 0.00029985002812265633] |
|
2024-11-27 22:28:19,573 example INFO Train Epoch: 146 [54%] |
|
2024-11-27 22:28:19,573 example INFO [2.6238412857055664, 2.3374087810516357, 8.5864839553833, 17.643659591674805, 1.9585093259811401, 2.5280542373657227, 60400, 0.00029985002812265633] |
|
2024-11-27 22:32:34,560 example INFO ====> Epoch: 146 |
|
2024-11-27 22:32:49,356 example INFO Train Epoch: 147 [2%] |
|
2024-11-27 22:32:49,356 example INFO [2.2971532344818115, 2.698820114135742, 9.242850303649902, 17.47536849975586, 2.0169544219970703, 1.9768656492233276, 60600, 0.000299812546869141] |
|
2024-11-27 22:37:17,379 example INFO Train Epoch: 147 [51%] |
|
2024-11-27 22:37:17,380 example INFO [2.310467481613159, 2.5886876583099365, 8.628618240356445, 17.071735382080078, 2.050161361694336, 1.8181536197662354, 60800, 0.000299812546869141] |
|
2024-11-27 22:39:28,015 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-27 22:39:28,016 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-27 22:39:34,559 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-27 22:39:34,560 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-27 22:39:34,592 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-27 22:39:34,704 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-27 22:39:34,763 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-27 22:39:34,767 example INFO Loaded checkpoint './logs/example/DUR_60000.pth' (iteration 145) |
|
2024-11-27 22:39:35,133 example INFO Loaded checkpoint './logs/example/G_60000.pth' (iteration 145) |
|
2024-11-27 22:39:35,252 example INFO Loaded checkpoint './logs/example/D_60000.pth' (iteration 145) |
|
2024-11-27 22:46:06,289 example INFO Train Epoch: 145 [10%] |
|
2024-11-27 22:46:06,290 example INFO [2.055534839630127, 2.8180460929870605, 9.378572463989258, 17.695636749267578, 1.9110465049743652, 2.146627902984619, 59800, 0.0002999625] |
|
2024-11-27 23:00:38,784 example INFO Train Epoch: 145 [58%] |
|
2024-11-27 23:00:38,784 example INFO [2.240830659866333, 2.6913037300109863, 8.11093807220459, 17.613401412963867, 2.0363457202911377, 2.0402772426605225, 60000, 0.0002999625] |
|
2024-11-27 23:00:43,141 example INFO Saving model and optimizer state at iteration 145 to ./logs/example/G_60000.pth |
|
2024-11-27 23:00:44,338 example INFO Saving model and optimizer state at iteration 145 to ./logs/example/D_60000.pth |
|
2024-11-27 23:00:45,082 example INFO Saving model and optimizer state at iteration 145 to ./logs/example/DUR_60000.pth |
|
2024-11-27 23:08:32,902 example INFO ====> Epoch: 145 |
|
2024-11-27 23:09:18,579 example INFO Train Epoch: 146 [6%] |
|
2024-11-27 23:09:18,579 example INFO [2.2330002784729004, 2.5743958950042725, 8.958558082580566, 17.777931213378906, 2.0165915489196777, 1.9136035442352295, 60200, 0.0002999250046875] |
|
2024-11-27 23:15:44,103 example INFO Train Epoch: 146 [54%] |
|
2024-11-27 23:15:44,103 example INFO [2.161303997039795, 2.7815256118774414, 8.970186233520508, 17.919403076171875, 1.9740276336669922, 2.00102162361145, 60400, 0.0002999250046875] |
|
2024-11-27 23:21:05,802 example INFO ====> Epoch: 146 |
|
2024-11-27 23:21:22,142 example INFO Train Epoch: 147 [2%] |
|
2024-11-27 23:21:22,142 example INFO [2.321866273880005, 2.4840927124023438, 8.431355476379395, 17.963459014892578, 2.02903151512146, 1.9048709869384766, 60600, 0.00029988751406191406] |
|
2024-11-27 23:26:28,677 example INFO Train Epoch: 147 [51%] |
|
2024-11-27 23:26:28,678 example INFO [2.439997911453247, 2.573427677154541, 8.688772201538086, 17.448238372802734, 1.954317569732666, 1.8040010929107666, 60800, 0.00029988751406191406] |
|
2024-11-27 23:31:23,770 example INFO Train Epoch: 147 [99%] |
|
2024-11-27 23:31:23,771 example INFO [2.272522211074829, 2.7305386066436768, 8.990506172180176, 18.595277786254883, 1.845597267150879, 2.290158271789551, 61000, 0.00029988751406191406] |
|
2024-11-27 23:31:27,259 example INFO Saving model and optimizer state at iteration 147 to ./logs/example/G_61000.pth |
|
2024-11-27 23:31:28,157 example INFO Saving model and optimizer state at iteration 147 to ./logs/example/D_61000.pth |
|
2024-11-27 23:31:28,661 example INFO Saving model and optimizer state at iteration 147 to ./logs/example/DUR_61000.pth |
|
2024-11-27 23:31:28,765 example INFO .. Free up space by deleting ckpt ./logs/example/G_56000.pth |
|
2024-11-27 23:31:28,766 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_58000.pth |
|
2024-11-27 23:31:28,856 example INFO .. Free up space by deleting ckpt ./logs/example/D_59000.pth |
|
2024-11-27 23:31:34,012 example INFO ====> Epoch: 147 |
|
2024-11-27 23:36:10,855 example INFO Train Epoch: 148 [47%] |
|
2024-11-27 23:36:10,856 example INFO [2.373161554336548, 2.6139492988586426, 8.562585830688477, 16.249351501464844, 1.9646403789520264, 2.067840337753296, 61200, 0.00029985002812265633] |
|
2024-11-27 23:41:00,276 example INFO Train Epoch: 148 [95%] |
|
2024-11-27 23:41:00,277 example INFO [2.421241044998169, 2.531951427459717, 9.50046157836914, 17.931215286254883, 1.834519863128662, 2.1133644580841064, 61400, 0.00029985002812265633] |
|
2024-11-27 23:41:26,768 example INFO ====> Epoch: 148 |
|
2024-11-27 23:45:51,456 example INFO Train Epoch: 149 [43%] |
|
2024-11-27 23:45:51,456 example INFO [2.342060089111328, 2.5165317058563232, 9.393684387207031, 17.58628273010254, 2.0870745182037354, 2.1641054153442383, 61600, 0.000299812546869141] |
|
2024-11-27 23:50:52,454 example INFO Train Epoch: 149 [92%] |
|
2024-11-27 23:50:52,454 example INFO [2.458688259124756, 2.7453207969665527, 8.768390655517578, 18.0830078125, 1.90213143825531, 1.8082026243209839, 61800, 0.000299812546869141] |
|
2024-11-27 23:51:41,508 example INFO ====> Epoch: 149 |
|
2024-11-27 23:55:43,418 example INFO Train Epoch: 150 [40%] |
|
2024-11-27 23:55:43,418 example INFO [2.038140296936035, 2.995595693588257, 8.822598457336426, 17.52618980407715, 1.9146935939788818, 2.0254039764404297, 62000, 0.00029977507030078235] |
|
2024-11-27 23:55:46,446 example INFO Saving model and optimizer state at iteration 150 to ./logs/example/G_62000.pth |
|
2024-11-27 23:55:47,340 example INFO Saving model and optimizer state at iteration 150 to ./logs/example/D_62000.pth |
|
2024-11-27 23:55:47,889 example INFO Saving model and optimizer state at iteration 150 to ./logs/example/DUR_62000.pth |
|
2024-11-27 23:55:48,000 example INFO .. Free up space by deleting ckpt ./logs/example/G_57000.pth |
|
2024-11-27 23:55:48,001 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_59000.pth |
|
2024-11-27 23:55:48,101 example INFO .. Free up space by deleting ckpt ./logs/example/D_60000.pth |
|
2024-11-28 00:00:30,427 example INFO Train Epoch: 150 [88%] |
|
2024-11-28 00:00:30,428 example INFO [2.1473913192749023, 2.7869927883148193, 7.984297275543213, 18.35161018371582, 2.01051664352417, 1.9694106578826904, 62200, 0.00029977507030078235] |
|
2024-11-28 00:01:39,489 example INFO ====> Epoch: 150 |
|
2024-11-28 00:05:06,163 example INFO Train Epoch: 151 [36%] |
|
2024-11-28 00:05:06,164 example INFO [2.2727713584899902, 2.5915260314941406, 9.725467681884766, 17.796266555786133, 1.9478415250778198, 1.7696354389190674, 62400, 0.00029973759841699473] |
|
2024-11-28 00:09:45,173 example INFO Train Epoch: 151 [84%] |
|
2024-11-28 00:09:45,173 example INFO [2.162986993789673, 3.147494316101074, 9.386734962463379, 18.972942352294922, 2.0707786083221436, 2.043635129928589, 62600, 0.00029973759841699473] |
|
2024-11-28 00:11:11,648 example INFO ====> Epoch: 151 |
|
2024-11-28 00:14:15,638 example INFO Train Epoch: 152 [33%] |
|
2024-11-28 00:14:15,639 example INFO [1.9618244171142578, 2.9663238525390625, 9.51779842376709, 17.25585174560547, 1.9136114120483398, 1.5629510879516602, 62800, 0.00029970013121719257] |
|
2024-11-28 00:19:03,520 example INFO Train Epoch: 152 [81%] |
|
2024-11-28 00:19:03,520 example INFO [2.322150707244873, 2.800936698913574, 7.541131973266602, 17.043922424316406, 2.0410571098327637, 2.005396604537964, 63000, 0.00029970013121719257] |
|
2024-11-28 00:19:07,156 example INFO Saving model and optimizer state at iteration 152 to ./logs/example/G_63000.pth |
|
2024-11-28 00:19:08,031 example INFO Saving model and optimizer state at iteration 152 to ./logs/example/D_63000.pth |
|
2024-11-28 00:19:08,578 example INFO Saving model and optimizer state at iteration 152 to ./logs/example/DUR_63000.pth |
|
2024-11-28 00:19:08,691 example INFO .. Free up space by deleting ckpt ./logs/example/G_58000.pth |
|
2024-11-28 00:19:08,693 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_60000.pth |
|
2024-11-28 00:19:08,790 example INFO .. Free up space by deleting ckpt ./logs/example/D_61000.pth |
|
2024-11-28 00:21:05,884 example INFO ====> Epoch: 152 |
|
2024-11-28 00:24:06,230 example INFO Train Epoch: 153 [29%] |
|
2024-11-28 00:24:06,230 example INFO [2.334564685821533, 2.723804473876953, 9.688865661621094, 17.540437698364258, 1.9317877292633057, 1.895324945449829, 63200, 0.0002996626687007904] |
|
2024-11-28 00:29:12,328 example INFO Train Epoch: 153 [77%] |
|
2024-11-28 00:29:12,329 example INFO [2.142432451248169, 2.80550479888916, 9.827794075012207, 16.971466064453125, 1.8051873445510864, 1.7783238887786865, 63400, 0.0002996626687007904] |
|
2024-11-28 00:31:35,888 example INFO ====> Epoch: 153 |
|
2024-11-28 00:34:06,070 example INFO Train Epoch: 154 [25%] |
|
2024-11-28 00:34:06,071 example INFO [2.280017137527466, 2.654391050338745, 8.745859146118164, 17.438796997070312, 1.7767093181610107, 1.7155356407165527, 63600, 0.0002996252108672028] |
|
2024-11-28 00:38:51,608 example INFO Train Epoch: 154 [73%] |
|
2024-11-28 00:38:51,608 example INFO [2.254851818084717, 2.536759376525879, 8.154715538024902, 17.871047973632812, 2.023160219192505, 1.9148329496383667, 63800, 0.0002996252108672028] |
|
2024-11-28 00:41:22,993 example INFO ====> Epoch: 154 |
|
2024-11-28 00:43:33,759 example INFO Train Epoch: 155 [22%] |
|
2024-11-28 00:43:33,759 example INFO [2.3834216594696045, 2.5400466918945312, 7.742903232574463, 17.181182861328125, 1.9444236755371094, 2.020977020263672, 64000, 0.00029958775771584436] |
|
2024-11-28 00:43:37,570 example INFO Saving model and optimizer state at iteration 155 to ./logs/example/G_64000.pth |
|
2024-11-28 00:43:38,496 example INFO Saving model and optimizer state at iteration 155 to ./logs/example/D_64000.pth |
|
2024-11-28 00:43:39,040 example INFO Saving model and optimizer state at iteration 155 to ./logs/example/DUR_64000.pth |
|
2024-11-28 00:43:39,153 example INFO .. Free up space by deleting ckpt ./logs/example/G_59000.pth |
|
2024-11-28 00:43:39,154 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_61000.pth |
|
2024-11-28 00:43:39,251 example INFO .. Free up space by deleting ckpt ./logs/example/D_62000.pth |
|
2024-11-28 00:48:11,714 example INFO Train Epoch: 155 [70%] |
|
2024-11-28 00:48:11,714 example INFO [2.55145001411438, 2.598226308822632, 8.244114875793457, 18.330810546875, 2.0622482299804688, 1.9090698957443237, 64200, 0.00029958775771584436] |
|
2024-11-28 00:51:19,354 example INFO ====> Epoch: 155 |
|
2024-11-28 00:53:05,428 example INFO Train Epoch: 156 [18%] |
|
2024-11-28 00:53:05,429 example INFO [2.504969358444214, 2.2952163219451904, 7.2036824226379395, 16.602251052856445, 1.9982948303222656, 1.1468113660812378, 64400, 0.0002995503092461299] |
|
2024-11-28 00:57:44,115 example INFO Train Epoch: 156 [66%] |
|
2024-11-28 00:57:44,116 example INFO [2.3992109298706055, 2.326056718826294, 8.550406455993652, 17.29253387451172, 2.0401229858398438, 1.9433438777923584, 64600, 0.0002995503092461299] |
|
2024-11-28 01:00:50,956 example INFO ====> Epoch: 156 |
|
2024-11-28 01:02:16,873 example INFO Train Epoch: 157 [14%] |
|
2024-11-28 01:02:16,873 example INFO [2.265744209289551, 3.08219838142395, 8.001175880432129, 17.028697967529297, 1.9043169021606445, 1.660017728805542, 64800, 0.0002995128654574741] |
|
2024-11-28 01:07:05,764 example INFO Train Epoch: 157 [63%] |
|
2024-11-28 01:07:05,764 example INFO [2.449486255645752, 2.3170924186706543, 8.73084831237793, 17.54559898376465, 1.8797967433929443, 1.90524423122406, 65000, 0.0002995128654574741] |
|
2024-11-28 01:07:08,957 example INFO Saving model and optimizer state at iteration 157 to ./logs/example/G_65000.pth |
|
2024-11-28 01:07:09,836 example INFO Saving model and optimizer state at iteration 157 to ./logs/example/D_65000.pth |
|
2024-11-28 01:07:10,338 example INFO Saving model and optimizer state at iteration 157 to ./logs/example/DUR_65000.pth |
|
2024-11-28 01:07:10,440 example INFO .. Free up space by deleting ckpt ./logs/example/G_60000.pth |
|
2024-11-28 01:07:10,442 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_62000.pth |
|
2024-11-28 01:07:10,530 example INFO .. Free up space by deleting ckpt ./logs/example/D_63000.pth |
|
2024-11-28 01:10:43,957 example INFO ====> Epoch: 157 |
|
2024-11-28 01:11:50,913 example INFO Train Epoch: 158 [11%] |
|
2024-11-28 01:11:50,914 example INFO [2.010218620300293, 3.1277072429656982, 10.201750755310059, 17.974885940551758, 1.8490581512451172, 2.3642218112945557, 65200, 0.0002994754263492919] |
|
2024-11-28 01:16:31,399 example INFO Train Epoch: 158 [59%] |
|
2024-11-28 01:16:31,400 example INFO [2.114497661590576, 2.8393399715423584, 8.68166446685791, 18.008440017700195, 2.0222413539886475, 1.9342174530029297, 65400, 0.0002994754263492919] |
|
2024-11-28 01:20:28,474 example INFO ====> Epoch: 158 |
|
2024-11-28 01:21:13,649 example INFO Train Epoch: 159 [7%] |
|
2024-11-28 01:21:13,650 example INFO [2.1977481842041016, 2.8490774631500244, 9.47175407409668, 17.10967254638672, 2.0214288234710693, 1.8769772052764893, 65600, 0.00029943799192099825] |
|
2024-11-28 01:25:49,986 example INFO Train Epoch: 159 [55%] |
|
2024-11-28 01:25:49,987 example INFO [2.441910743713379, 2.7920427322387695, 8.01303768157959, 17.337656021118164, 2.0827689170837402, 2.0121376514434814, 65800, 0.00029943799192099825] |
|
2024-11-28 01:30:13,617 example INFO ====> Epoch: 159 |
|
2024-11-28 01:30:37,163 example INFO Train Epoch: 160 [4%] |
|
2024-11-28 01:30:37,164 example INFO [2.1369917392730713, 2.583024501800537, 9.797451972961426, 17.243515014648438, 1.7684321403503418, 1.8027052879333496, 66000, 0.0002994005621720081] |
|
2024-11-28 01:30:40,598 example INFO Saving model and optimizer state at iteration 160 to ./logs/example/G_66000.pth |
|
2024-11-28 01:30:42,400 example INFO Saving model and optimizer state at iteration 160 to ./logs/example/D_66000.pth |
|
2024-11-28 01:30:42,942 example INFO Saving model and optimizer state at iteration 160 to ./logs/example/DUR_66000.pth |
|
2024-11-28 01:30:43,053 example INFO .. Free up space by deleting ckpt ./logs/example/G_61000.pth |
|
2024-11-28 01:30:43,054 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_63000.pth |
|
2024-11-28 01:30:43,151 example INFO .. Free up space by deleting ckpt ./logs/example/D_64000.pth |
|
2024-11-28 01:35:13,325 example INFO Train Epoch: 160 [52%] |
|
2024-11-28 01:35:13,325 example INFO [2.416884422302246, 2.613269805908203, 9.372631072998047, 17.74479103088379, 1.9819791316986084, 1.9953352212905884, 66200, 0.0002994005621720081] |
|
2024-11-28 01:39:50,136 example INFO ====> Epoch: 160 |
|
2024-11-28 01:39:52,932 example INFO Train Epoch: 161 [0%] |
|
2024-11-28 01:39:52,933 example INFO [2.0948987007141113, 2.7243618965148926, 9.251015663146973, 17.86430549621582, 1.7555369138717651, 1.7104719877243042, 66400, 0.0002993631371017366] |
|
2024-11-28 01:44:36,094 example INFO Train Epoch: 161 [48%] |
|
2024-11-28 01:44:36,094 example INFO [2.041433811187744, 2.770298480987549, 8.842094421386719, 17.809471130371094, 2.037536859512329, 1.979008436203003, 66600, 0.0002993631371017366] |
|
2024-11-28 01:49:15,686 example INFO Train Epoch: 161 [96%] |
|
2024-11-28 01:49:15,687 example INFO [2.1675496101379395, 2.640526294708252, 8.74881362915039, 17.41883087158203, 2.0095481872558594, 1.9143941402435303, 66800, 0.0002993631371017366] |
|
2024-11-28 01:49:33,771 example INFO ====> Epoch: 161 |
|
2024-11-28 01:53:46,433 example INFO Train Epoch: 162 [45%] |
|
2024-11-28 01:53:46,434 example INFO [2.3292641639709473, 2.7296628952026367, 8.098184585571289, 18.494243621826172, 1.9445216655731201, 2.1188716888427734, 67000, 0.00029932571670959887] |
|
2024-11-28 01:53:50,011 example INFO Saving model and optimizer state at iteration 162 to ./logs/example/G_67000.pth |
|
2024-11-28 01:53:50,892 example INFO Saving model and optimizer state at iteration 162 to ./logs/example/D_67000.pth |
|
2024-11-28 01:53:51,437 example INFO Saving model and optimizer state at iteration 162 to ./logs/example/DUR_67000.pth |
|
2024-11-28 01:53:51,550 example INFO .. Free up space by deleting ckpt ./logs/example/G_62000.pth |
|
2024-11-28 01:53:51,551 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_64000.pth |
|
2024-11-28 01:53:51,650 example INFO .. Free up space by deleting ckpt ./logs/example/D_65000.pth |
|
2024-11-28 01:58:16,348 example INFO Train Epoch: 162 [93%] |
|
2024-11-28 01:58:16,349 example INFO [2.298797369003296, 2.778795003890991, 8.226181030273438, 17.20661163330078, 1.6110172271728516, 1.441895842552185, 67200, 0.00029932571670959887] |
|
2024-11-28 01:58:55,880 example INFO ====> Epoch: 162 |
|
2024-11-28 02:02:55,409 example INFO Train Epoch: 163 [41%] |
|
2024-11-28 02:02:55,409 example INFO [2.217451810836792, 2.9450294971466064, 8.630844116210938, 17.956157684326172, 1.8447647094726562, 1.97017502784729, 67400, 0.00029928830099501015] |
|
2024-11-28 02:07:22,189 example INFO Train Epoch: 163 [89%] |
|
2024-11-28 02:07:22,189 example INFO [2.3828988075256348, 2.4295716285705566, 8.159266471862793, 16.765172958374023, 2.0426244735717773, 2.2651681900024414, 67600, 0.00029928830099501015] |
|
2024-11-28 02:08:23,477 example INFO ====> Epoch: 163 |
|
2024-11-28 02:12:01,786 example INFO Train Epoch: 164 [37%] |
|
2024-11-28 02:12:01,786 example INFO [2.673978328704834, 2.2773470878601074, 6.783448219299316, 17.294981002807617, 1.8324788808822632, 2.046753406524658, 67800, 0.00029925088995738576] |
|
2024-11-28 02:16:47,661 example INFO Train Epoch: 164 [86%] |
|
2024-11-28 02:16:47,662 example INFO [2.2850677967071533, 2.551154136657715, 8.190055847167969, 17.917327880859375, 2.0492477416992188, 1.997898817062378, 68000, 0.00029925088995738576] |
|
2024-11-28 02:16:51,118 example INFO Saving model and optimizer state at iteration 164 to ./logs/example/G_68000.pth |
|
2024-11-28 02:16:52,872 example INFO Saving model and optimizer state at iteration 164 to ./logs/example/D_68000.pth |
|
2024-11-28 02:16:53,420 example INFO Saving model and optimizer state at iteration 164 to ./logs/example/DUR_68000.pth |
|
2024-11-28 02:16:53,530 example INFO .. Free up space by deleting ckpt ./logs/example/G_63000.pth |
|
2024-11-28 02:16:53,531 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_65000.pth |
|
2024-11-28 02:16:53,629 example INFO .. Free up space by deleting ckpt ./logs/example/D_66000.pth |
|
2024-11-28 02:18:15,291 example INFO ====> Epoch: 164 |
|
2024-11-28 02:21:32,114 example INFO Train Epoch: 165 [34%] |
|
2024-11-28 02:21:32,115 example INFO [2.1582119464874268, 2.77266788482666, 8.60694408416748, 17.198650360107422, 1.7944269180297852, 2.2732620239257812, 68200, 0.0002992134835961411] |
|
2024-11-28 02:26:06,638 example INFO Train Epoch: 165 [82%] |
|
2024-11-28 02:26:06,639 example INFO [2.203598976135254, 2.9656379222869873, 9.403534889221191, 18.537565231323242, 1.9566706418991089, 2.0999839305877686, 68400, 0.0002992134835961411] |
|
2024-11-28 02:27:47,604 example INFO ====> Epoch: 165 |
|
2024-11-28 02:30:37,415 example INFO Train Epoch: 166 [30%] |
|
2024-11-28 02:30:37,416 example INFO [2.6683011054992676, 2.6372694969177246, 8.627982139587402, 17.577438354492188, 1.7212727069854736, 1.960312008857727, 68600, 0.00029917608191069156] |
|
2024-11-28 02:35:17,923 example INFO Train Epoch: 166 [78%] |
|
2024-11-28 02:35:17,924 example INFO [2.1454200744628906, 2.821316719055176, 8.202861785888672, 17.580238342285156, 1.966865062713623, 1.8989713191986084, 68800, 0.00029917608191069156] |
|
2024-11-28 02:37:18,574 example INFO ====> Epoch: 166 |
|
2024-11-28 02:40:00,982 example INFO Train Epoch: 167 [27%] |
|
2024-11-28 02:40:00,982 example INFO [2.381183385848999, 2.5419418811798096, 8.105152130126953, 18.326492309570312, 2.0181145668029785, 2.29093074798584, 69000, 0.0002991386849004527] |
|
2024-11-28 02:40:03,965 example INFO Saving model and optimizer state at iteration 167 to ./logs/example/G_69000.pth |
|
2024-11-28 02:40:04,842 example INFO Saving model and optimizer state at iteration 167 to ./logs/example/D_69000.pth |
|
2024-11-28 02:40:05,375 example INFO Saving model and optimizer state at iteration 167 to ./logs/example/DUR_69000.pth |
|
2024-11-28 02:40:05,485 example INFO .. Free up space by deleting ckpt ./logs/example/G_64000.pth |
|
2024-11-28 02:40:05,487 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_66000.pth |
|
2024-11-28 02:40:05,583 example INFO .. Free up space by deleting ckpt ./logs/example/D_67000.pth |
|
2024-11-28 02:44:37,864 example INFO Train Epoch: 167 [75%] |
|
2024-11-28 02:44:37,864 example INFO [2.0936832427978516, 2.8206701278686523, 9.844243049621582, 17.474227905273438, 1.9275068044662476, 2.0041141510009766, 69200, 0.0002991386849004527] |
|
2024-11-28 02:46:54,406 example INFO ====> Epoch: 167 |
|
2024-11-28 02:49:09,529 example INFO Train Epoch: 168 [23%] |
|
2024-11-28 02:49:09,530 example INFO [2.305196523666382, 3.0490612983703613, 9.010830879211426, 16.8936824798584, 1.8971869945526123, 1.6376433372497559, 69400, 0.00029910129256484016] |
|
2024-11-28 02:53:49,545 example INFO Train Epoch: 168 [71%] |
|
2024-11-28 02:53:49,545 example INFO [2.1113686561584473, 2.8832638263702393, 8.721197128295898, 17.221803665161133, 1.7555201053619385, 2.074232339859009, 69600, 0.00029910129256484016] |
|
2024-11-28 02:56:31,376 example INFO ====> Epoch: 168 |
|
2024-11-28 02:58:28,398 example INFO Train Epoch: 169 [19%] |
|
2024-11-28 02:58:28,398 example INFO [2.4326586723327637, 2.580435037612915, 7.024168968200684, 17.12211036682129, 1.9192607402801514, 2.0601587295532227, 69800, 0.0002990639049032695] |
|
2024-11-28 03:03:06,605 example INFO Train Epoch: 169 [67%] |
|
2024-11-28 03:03:06,605 example INFO [2.152414083480835, 3.0932834148406982, 9.9533109664917, 18.292604446411133, 2.071614980697632, 1.7053757905960083, 70000, 0.0002990639049032695] |
|
2024-11-28 03:03:10,250 example INFO Saving model and optimizer state at iteration 169 to ./logs/example/G_70000.pth |
|
2024-11-28 03:03:11,140 example INFO Saving model and optimizer state at iteration 169 to ./logs/example/D_70000.pth |
|
2024-11-28 03:03:11,685 example INFO Saving model and optimizer state at iteration 169 to ./logs/example/DUR_70000.pth |
|
2024-11-28 03:03:11,797 example INFO .. Free up space by deleting ckpt ./logs/example/G_65000.pth |
|
2024-11-28 03:03:11,798 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_67000.pth |
|
2024-11-28 03:03:11,896 example INFO .. Free up space by deleting ckpt ./logs/example/D_68000.pth |
|
2024-11-28 03:06:22,217 example INFO ====> Epoch: 169 |
|
2024-11-28 03:07:59,326 example INFO Train Epoch: 170 [16%] |
|
2024-11-28 03:07:59,327 example INFO [2.3024041652679443, 2.486936092376709, 8.463871002197266, 17.270050048828125, 2.005506992340088, 1.7046228647232056, 70200, 0.0002990265219151566] |
|
2024-11-28 03:12:31,987 example INFO Train Epoch: 170 [64%] |
|
2024-11-28 03:12:31,987 example INFO [2.430893898010254, 2.528930187225342, 8.282483100891113, 16.735057830810547, 1.9069422483444214, 2.2022852897644043, 70400, 0.0002990265219151566] |
|
2024-11-28 03:15:50,677 example INFO ====> Epoch: 170 |
|
2024-11-28 03:17:03,205 example INFO Train Epoch: 171 [12%] |
|
2024-11-28 03:17:03,206 example INFO [2.4239721298217773, 2.640363931655884, 7.614161968231201, 17.506765365600586, 1.9875240325927734, 1.9445979595184326, 70600, 0.00029898914359991716] |
|
2024-11-28 03:21:37,494 example INFO Train Epoch: 171 [60%] |
|
2024-11-28 03:21:37,495 example INFO [2.4334330558776855, 2.6653337478637695, 7.880756378173828, 17.043865203857422, 1.927098035812378, 2.0085577964782715, 70800, 0.00029898914359991716] |
|
2024-11-28 03:25:12,514 example INFO ====> Epoch: 171 |
|
2024-11-28 03:26:04,232 example INFO Train Epoch: 172 [8%] |
|
2024-11-28 03:26:04,232 example INFO [2.1089770793914795, 2.9802052974700928, 9.976587295532227, 17.394508361816406, 1.856513500213623, 1.519690990447998, 71000, 0.00029895176995696715] |
|
2024-11-28 03:26:06,501 example INFO Saving model and optimizer state at iteration 172 to ./logs/example/G_71000.pth |
|
2024-11-28 03:26:07,388 example INFO Saving model and optimizer state at iteration 172 to ./logs/example/D_71000.pth |
|
2024-11-28 03:26:07,881 example INFO Saving model and optimizer state at iteration 172 to ./logs/example/DUR_71000.pth |
|
2024-11-28 03:26:07,983 example INFO .. Free up space by deleting ckpt ./logs/example/G_66000.pth |
|
2024-11-28 03:26:07,984 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_68000.pth |
|
2024-11-28 03:26:08,072 example INFO .. Free up space by deleting ckpt ./logs/example/D_69000.pth |
|
2024-11-28 03:30:43,444 example INFO Train Epoch: 172 [57%] |
|
2024-11-28 03:30:43,444 example INFO [2.3867383003234863, 2.7388856410980225, 8.833447456359863, 17.299625396728516, 1.9864674806594849, 1.9634168148040771, 71200, 0.00029895176995696715] |
|
2024-11-28 03:34:49,641 example INFO ====> Epoch: 172 |
|
2024-11-28 03:35:21,185 example INFO Train Epoch: 173 [5%] |
|
2024-11-28 03:35:21,186 example INFO [2.5096187591552734, 2.558004379272461, 8.848251342773438, 17.309188842773438, 1.8191273212432861, 1.5554180145263672, 71400, 0.0002989144009857225] |
|
2024-11-28 03:40:05,755 example INFO Train Epoch: 173 [53%] |
|
2024-11-28 03:40:05,755 example INFO [2.012817859649658, 2.8741726875305176, 10.180020332336426, 17.310134887695312, 1.9526013135910034, 1.9582061767578125, 71600, 0.0002989144009857225] |
|
2024-11-28 03:44:45,537 example INFO ====> Epoch: 173 |
|
2024-11-28 03:44:54,804 example INFO Train Epoch: 174 [1%] |
|
2024-11-28 03:44:54,805 example INFO [2.2902603149414062, 2.4689931869506836, 8.39016056060791, 16.979145050048828, 1.9244177341461182, 1.6901041269302368, 71800, 0.0002988770366855993] |
|
2024-11-28 03:49:37,663 example INFO Train Epoch: 174 [49%] |
|
2024-11-28 03:49:37,663 example INFO [2.063269853591919, 2.6988205909729004, 9.587325096130371, 16.677814483642578, 1.9680255651474, 1.6972495317459106, 72000, 0.0002988770366855993] |
|
2024-11-28 03:49:39,969 example INFO Saving model and optimizer state at iteration 174 to ./logs/example/G_72000.pth |
|
2024-11-28 03:49:41,353 example INFO Saving model and optimizer state at iteration 174 to ./logs/example/D_72000.pth |
|
2024-11-28 03:49:41,847 example INFO Saving model and optimizer state at iteration 174 to ./logs/example/DUR_72000.pth |
|
2024-11-28 03:49:41,950 example INFO .. Free up space by deleting ckpt ./logs/example/G_67000.pth |
|
2024-11-28 03:49:41,951 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_69000.pth |
|
2024-11-28 03:49:42,039 example INFO .. Free up space by deleting ckpt ./logs/example/D_70000.pth |
|
2024-11-28 03:54:14,153 example INFO Train Epoch: 174 [98%] |
|
2024-11-28 03:54:14,154 example INFO [2.138746738433838, 2.7650198936462402, 8.866204261779785, 17.30976104736328, 2.0393314361572266, 1.5344873666763306, 72200, 0.0002988770366855993] |
|
2024-11-28 03:54:25,980 example INFO ====> Epoch: 174 |
|
2024-11-28 03:58:54,275 example INFO Train Epoch: 175 [46%] |
|
2024-11-28 03:58:54,276 example INFO [2.1507084369659424, 2.8603556156158447, 9.238663673400879, 17.424320220947266, 2.113076686859131, 1.8844795227050781, 72400, 0.0002988396770560136] |
|
2024-11-28 04:03:32,423 example INFO Train Epoch: 175 [94%] |
|
2024-11-28 04:03:32,423 example INFO [2.2262916564941406, 2.574921131134033, 8.558899879455566, 17.378904342651367, 1.9594073295593262, 1.8127118349075317, 72600, 0.0002988396770560136] |
|
2024-11-28 04:04:07,649 example INFO ====> Epoch: 175 |
|
2024-11-28 04:08:17,700 example INFO Train Epoch: 176 [42%] |
|
2024-11-28 04:08:17,701 example INFO [2.093949794769287, 2.808180332183838, 9.497586250305176, 17.75954246520996, 1.9886553287506104, 1.9739112854003906, 72800, 0.00029880232209638155] |
|
2024-11-28 04:13:09,700 example INFO Train Epoch: 176 [90%] |
|
2024-11-28 04:13:09,701 example INFO [2.2966649532318115, 2.7638566493988037, 9.334905624389648, 17.58514976501465, 1.7302634716033936, 1.695299506187439, 73000, 0.00029880232209638155] |
|
2024-11-28 04:13:12,787 example INFO Saving model and optimizer state at iteration 176 to ./logs/example/G_73000.pth |
|
2024-11-28 04:13:13,685 example INFO Saving model and optimizer state at iteration 176 to ./logs/example/D_73000.pth |
|
2024-11-28 04:13:14,239 example INFO Saving model and optimizer state at iteration 176 to ./logs/example/DUR_73000.pth |
|
2024-11-28 04:13:14,352 example INFO .. Free up space by deleting ckpt ./logs/example/G_68000.pth |
|
2024-11-28 04:13:14,353 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_70000.pth |
|
2024-11-28 04:13:14,449 example INFO .. Free up space by deleting ckpt ./logs/example/D_71000.pth |
|
2024-11-28 04:14:11,466 example INFO ====> Epoch: 176 |
|
2024-11-28 04:18:06,568 example INFO Train Epoch: 177 [39%] |
|
2024-11-28 04:18:06,568 example INFO [2.299704074859619, 2.8685357570648193, 8.945741653442383, 17.7748966217041, 2.1091156005859375, 2.2705652713775635, 73200, 0.0002987649718061195] |
|
2024-11-28 04:22:36,006 example INFO Train Epoch: 177 [87%] |
|
2024-11-28 04:22:36,007 example INFO [2.0962696075439453, 2.764237403869629, 8.84914493560791, 17.199050903320312, 1.8382819890975952, 2.049992799758911, 73400, 0.0002987649718061195] |
|
2024-11-28 04:23:45,786 example INFO ====> Epoch: 177 |
|
2024-11-28 04:27:05,637 example INFO Train Epoch: 178 [35%] |
|
2024-11-28 04:27:05,637 example INFO [2.305933713912964, 2.6419155597686768, 8.35295581817627, 17.74579620361328, 1.8212385177612305, 1.6846891641616821, 73600, 0.00029872762618464373] |
|
2024-11-28 04:31:43,053 example INFO Train Epoch: 178 [83%] |
|
2024-11-28 04:31:43,054 example INFO [2.4044179916381836, 2.573183536529541, 8.539083480834961, 16.969070434570312, 2.069488048553467, 1.6292730569839478, 73800, 0.00029872762618464373] |
|
2024-11-28 04:33:16,793 example INFO ====> Epoch: 178 |
|
2024-11-28 04:36:19,415 example INFO Train Epoch: 179 [31%] |
|
2024-11-28 04:36:19,416 example INFO [2.2460992336273193, 2.7415566444396973, 8.39232063293457, 18.125600814819336, 1.9149771928787231, 2.1561367511749268, 74000, 0.00029869028523137064] |
|
2024-11-28 04:36:22,204 example INFO Saving model and optimizer state at iteration 179 to ./logs/example/G_74000.pth |
|
2024-11-28 04:36:23,939 example INFO Saving model and optimizer state at iteration 179 to ./logs/example/D_74000.pth |
|
2024-11-28 04:36:24,474 example INFO Saving model and optimizer state at iteration 179 to ./logs/example/DUR_74000.pth |
|
2024-11-28 04:36:24,585 example INFO .. Free up space by deleting ckpt ./logs/example/G_69000.pth |
|
2024-11-28 04:36:24,587 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_71000.pth |
|
2024-11-28 04:36:24,684 example INFO .. Free up space by deleting ckpt ./logs/example/D_72000.pth |
|
2024-11-28 04:40:46,294 example INFO Train Epoch: 179 [80%] |
|
2024-11-28 04:40:46,294 example INFO [2.1552677154541016, 2.9120163917541504, 9.994003295898438, 16.907825469970703, 1.8333158493041992, 1.864319920539856, 74200, 0.00029869028523137064] |
|
2024-11-28 04:42:46,163 example INFO ====> Epoch: 179 |
|
2024-11-28 04:45:30,345 example INFO Train Epoch: 180 [28%] |
|
2024-11-28 04:45:30,345 example INFO [2.0760555267333984, 2.885197877883911, 8.62219524383545, 17.44684410095215, 1.8594740629196167, 1.8617525100708008, 74400, 0.0002986529489457167] |
|
2024-11-28 04:50:16,596 example INFO Train Epoch: 180 [76%] |
|
2024-11-28 04:50:16,597 example INFO [2.511620044708252, 2.6091675758361816, 8.583274841308594, 17.20587730407715, 1.9048792123794556, 2.3822195529937744, 74600, 0.0002986529489457167] |
|
2024-11-28 04:52:33,794 example INFO ====> Epoch: 180 |
|
2024-11-28 04:55:02,528 example INFO Train Epoch: 181 [24%] |
|
2024-11-28 04:55:02,529 example INFO [2.01511812210083, 3.015263557434082, 10.322949409484863, 18.038307189941406, 1.882614016532898, 1.977035403251648, 74800, 0.00029861561732709847] |
|
2024-11-28 04:59:45,334 example INFO Train Epoch: 181 [72%] |
|
2024-11-28 04:59:45,334 example INFO [2.273190498352051, 2.863342761993408, 8.044132232666016, 17.043121337890625, 1.9622697830200195, 1.5643647909164429, 75000, 0.00029861561732709847] |
|
2024-11-28 04:59:47,406 example INFO Saving model and optimizer state at iteration 181 to ./logs/example/G_75000.pth |
|
2024-11-28 04:59:48,266 example INFO Saving model and optimizer state at iteration 181 to ./logs/example/D_75000.pth |
|
2024-11-28 04:59:48,802 example INFO Saving model and optimizer state at iteration 181 to ./logs/example/DUR_75000.pth |
|
2024-11-28 04:59:48,916 example INFO .. Free up space by deleting ckpt ./logs/example/G_70000.pth |
|
2024-11-28 04:59:48,917 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_72000.pth |
|
2024-11-28 04:59:49,017 example INFO .. Free up space by deleting ckpt ./logs/example/D_73000.pth |
|
2024-11-28 05:02:22,712 example INFO ====> Epoch: 181 |
|
2024-11-28 05:04:23,824 example INFO Train Epoch: 182 [20%] |
|
2024-11-28 05:04:23,825 example INFO [2.254300594329834, 2.6300978660583496, 9.1865816116333, 17.45627784729004, 2.0037684440612793, 2.034552574157715, 75200, 0.00029857829037493257] |
|
2024-11-28 05:08:55,935 example INFO Train Epoch: 182 [69%] |
|
2024-11-28 05:08:55,935 example INFO [1.971024513244629, 2.839789628982544, 9.673792839050293, 16.858781814575195, 1.9530766010284424, 1.7370291948318481, 75400, 0.00029857829037493257] |
|
2024-11-28 05:11:54,387 example INFO ====> Epoch: 182 |
|
2024-11-28 05:13:27,079 example INFO Train Epoch: 183 [17%] |
|
2024-11-28 05:13:27,080 example INFO [2.0740106105804443, 3.280372381210327, 8.792548179626465, 17.727956771850586, 1.9973170757293701, 2.162108898162842, 75600, 0.0002985409680886357] |
|
2024-11-28 05:17:57,011 example INFO Train Epoch: 183 [65%] |
|
2024-11-28 05:17:57,012 example INFO [2.3989405632019043, 2.481812000274658, 7.131719589233398, 17.19452476501465, 1.9809629917144775, 1.8704944849014282, 75800, 0.0002985409680886357] |
|
2024-11-28 05:21:10,975 example INFO ====> Epoch: 183 |
|
2024-11-28 05:22:28,362 example INFO Train Epoch: 184 [13%] |
|
2024-11-28 05:22:28,362 example INFO [2.317455768585205, 2.7217252254486084, 8.52471923828125, 18.39240074157715, 1.9235578775405884, 1.8425925970077515, 76000, 0.0002985036504676246] |
|
2024-11-28 05:22:31,313 example INFO Saving model and optimizer state at iteration 184 to ./logs/example/G_76000.pth |
|
2024-11-28 05:22:32,153 example INFO Saving model and optimizer state at iteration 184 to ./logs/example/D_76000.pth |
|
2024-11-28 05:22:32,657 example INFO Saving model and optimizer state at iteration 184 to ./logs/example/DUR_76000.pth |
|
2024-11-28 05:22:32,760 example INFO .. Free up space by deleting ckpt ./logs/example/G_71000.pth |
|
2024-11-28 05:22:32,761 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_73000.pth |
|
2024-11-28 05:22:32,849 example INFO .. Free up space by deleting ckpt ./logs/example/D_74000.pth |
|
2024-11-28 05:27:10,719 example INFO Train Epoch: 184 [61%] |
|
2024-11-28 05:27:10,719 example INFO [2.0962774753570557, 2.7501840591430664, 8.826979637145996, 17.250431060791016, 2.103848934173584, 1.916019320487976, 76200, 0.0002985036504676246] |
|
2024-11-28 05:30:48,356 example INFO ====> Epoch: 184 |
|
2024-11-28 05:31:49,843 example INFO Train Epoch: 185 [10%] |
|
2024-11-28 05:31:49,844 example INFO [2.2865889072418213, 2.5355775356292725, 6.4213104248046875, 17.948204040527344, 2.138347625732422, 2.136430025100708, 76400, 0.00029846633751131615] |
|
2024-11-28 05:36:29,000 example INFO Train Epoch: 185 [58%] |
|
2024-11-28 05:36:29,000 example INFO [1.8941686153411865, 3.168950080871582, 10.179460525512695, 18.525856018066406, 1.942568302154541, 2.0059659481048584, 76600, 0.00029846633751131615] |
|
2024-11-28 05:40:20,647 example INFO ====> Epoch: 185 |
|
2024-11-28 05:40:59,261 example INFO Train Epoch: 186 [6%] |
|
2024-11-28 05:40:59,262 example INFO [2.1336193084716797, 3.084841728210449, 9.430109977722168, 17.925403594970703, 2.0471560955047607, 1.8268237113952637, 76800, 0.00029842902921912724] |
|
2024-11-28 05:45:33,491 example INFO Train Epoch: 186 [54%] |
|
2024-11-28 05:45:33,492 example INFO [2.0742368698120117, 2.753443717956543, 8.967896461486816, 16.811290740966797, 2.0539674758911133, 2.2899622917175293, 77000, 0.00029842902921912724] |
|
2024-11-28 05:45:35,868 example INFO Saving model and optimizer state at iteration 186 to ./logs/example/G_77000.pth |
|
2024-11-28 05:45:36,803 example INFO Saving model and optimizer state at iteration 186 to ./logs/example/D_77000.pth |
|
2024-11-28 05:45:37,336 example INFO Saving model and optimizer state at iteration 186 to ./logs/example/DUR_77000.pth |
|
2024-11-28 05:45:37,448 example INFO .. Free up space by deleting ckpt ./logs/example/G_72000.pth |
|
2024-11-28 05:45:37,450 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_74000.pth |
|
2024-11-28 05:45:37,546 example INFO .. Free up space by deleting ckpt ./logs/example/D_75000.pth |
|
2024-11-28 05:49:53,342 example INFO ====> Epoch: 186 |
|
2024-11-28 05:50:10,392 example INFO Train Epoch: 187 [2%] |
|
2024-11-28 05:50:10,392 example INFO [2.3860349655151367, 2.707658052444458, 8.683338165283203, 17.16679573059082, 1.7077888250350952, 2.078709602355957, 77200, 0.0002983917255904748] |
|
2024-11-28 05:54:40,244 example INFO Train Epoch: 187 [51%] |
|
2024-11-28 05:54:40,245 example INFO [2.0971086025238037, 3.0552988052368164, 9.727290153503418, 17.754350662231445, 2.0145859718322754, 1.6027734279632568, 77400, 0.0002983917255904748] |
|
2024-11-28 05:59:13,752 example INFO Train Epoch: 187 [99%] |
|
2024-11-28 05:59:13,753 example INFO [2.1896464824676514, 2.676542282104492, 8.785247802734375, 17.525354385375977, 1.9206042289733887, 1.998317003250122, 77600, 0.0002983917255904748] |
|
2024-11-28 05:59:19,196 example INFO ====> Epoch: 187 |
|
2024-11-28 06:03:42,399 example INFO Train Epoch: 188 [47%] |
|
2024-11-28 06:03:42,399 example INFO [2.5515050888061523, 2.3159801959991455, 6.696722030639648, 17.213775634765625, 2.050562858581543, 2.0713155269622803, 77800, 0.000298354426624776] |
|
2024-11-28 06:08:17,949 example INFO Train Epoch: 188 [95%] |
|
2024-11-28 06:08:17,950 example INFO [2.5046303272247314, 2.364858627319336, 6.3817362785339355, 17.247770309448242, 1.8878018856048584, 2.404193878173828, 78000, 0.000298354426624776] |
|
2024-11-28 06:08:20,963 example INFO Saving model and optimizer state at iteration 188 to ./logs/example/G_78000.pth |
|
2024-11-28 06:08:22,714 example INFO Saving model and optimizer state at iteration 188 to ./logs/example/D_78000.pth |
|
2024-11-28 06:08:23,264 example INFO Saving model and optimizer state at iteration 188 to ./logs/example/DUR_78000.pth |
|
2024-11-28 06:08:23,377 example INFO .. Free up space by deleting ckpt ./logs/example/G_73000.pth |
|
2024-11-28 06:08:23,378 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_75000.pth |
|
2024-11-28 06:08:23,475 example INFO .. Free up space by deleting ckpt ./logs/example/D_76000.pth |
|
2024-11-28 06:08:48,271 example INFO ====> Epoch: 188 |
|
2024-11-28 06:12:55,867 example INFO Train Epoch: 189 [43%] |
|
2024-11-28 06:12:55,868 example INFO [2.592048168182373, 2.3224103450775146, 8.182469367980957, 16.73550796508789, 2.060920000076294, 2.4667983055114746, 78200, 0.0002983171323214479] |
|
2024-11-28 06:17:25,853 example INFO Train Epoch: 189 [92%] |
|
2024-11-28 06:17:25,853 example INFO [2.3234610557556152, 2.6152541637420654, 8.413177490234375, 17.74273681640625, 2.0188040733337402, 1.9589276313781738, 78400, 0.0002983171323214479] |
|
2024-11-28 06:18:11,961 example INFO ====> Epoch: 189 |
|
2024-11-28 06:22:12,043 example INFO Train Epoch: 190 [40%] |
|
2024-11-28 06:22:12,043 example INFO [2.4661922454833984, 2.3043558597564697, 6.60924768447876, 17.594518661499023, 2.064349412918091, 1.6638137102127075, 78600, 0.0002982798426799077] |
|
2024-11-28 06:26:39,429 example INFO Train Epoch: 190 [88%] |
|
2024-11-28 06:26:39,430 example INFO [2.2974953651428223, 2.863281726837158, 8.969080924987793, 18.303001403808594, 1.8675861358642578, 1.7704856395721436, 78800, 0.0002982798426799077] |
|
2024-11-28 06:27:45,554 example INFO ====> Epoch: 190 |
|
2024-11-28 06:31:19,997 example INFO Train Epoch: 191 [36%] |
|
2024-11-28 06:31:19,998 example INFO [2.5066280364990234, 2.2597336769104004, 8.951096534729004, 17.46186065673828, 1.9098069667816162, 1.4768321514129639, 79000, 0.0002982425576995727] |
|
2024-11-28 06:31:23,035 example INFO Saving model and optimizer state at iteration 191 to ./logs/example/G_79000.pth |
|
2024-11-28 06:31:23,916 example INFO Saving model and optimizer state at iteration 191 to ./logs/example/D_79000.pth |
|
2024-11-28 06:31:24,454 example INFO Saving model and optimizer state at iteration 191 to ./logs/example/DUR_79000.pth |
|
2024-11-28 06:31:24,567 example INFO .. Free up space by deleting ckpt ./logs/example/G_74000.pth |
|
2024-11-28 06:31:24,568 example INFO .. Free up space by deleting ckpt ./logs/example/DUR_76000.pth |
|
2024-11-28 06:31:24,665 example INFO .. Free up space by deleting ckpt ./logs/example/D_77000.pth |
|
2024-11-28 06:35:53,341 example INFO Train Epoch: 191 [84%] |
|
2024-11-28 06:35:53,342 example INFO [2.3031063079833984, 2.59881329536438, 9.499303817749023, 16.99353790283203, 1.9450033903121948, 1.8386898040771484, 79200, 0.0002982425576995727] |
|
2024-11-28 06:37:21,749 example INFO ====> Epoch: 191 |
|
2024-11-28 06:40:32,871 example INFO Train Epoch: 192 [33%] |
|
2024-11-28 06:40:32,871 example INFO [2.614213228225708, 2.188903570175171, 7.340160846710205, 18.026168823242188, 1.9492199420928955, 2.1982531547546387, 79400, 0.00029820527737986025] |
|
2024-11-28 06:45:18,956 example INFO Train Epoch: 192 [81%] |
|
2024-11-28 06:45:18,957 example INFO [2.201856851577759, 2.857673168182373, 9.513009071350098, 17.16515350341797, 1.8977243900299072, 1.8692597150802612, 79600, 0.00029820527737986025] |
|
2024-11-28 06:47:11,666 example INFO ====> Epoch: 192 |
|
2024-11-28 06:50:27,012 example INFO {'train': {'log_interval': 200, 'eval_interval': 1000, 'seed': 52, 'epochs': 10000, 'learning_rate': 0.0003, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 16384, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'skip_optimizer': True}, 'data': {'training_files': 'data/example/train.list', 'validation_files': 'data/example/val.list', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 128, 'mel_fmin': 0.0, 'mel_fmax': None, 'add_blank': True, 'n_speakers': 1, 'cleaned_text': True, 'spk2id': {'EN-default': 0}}, 'model': {'use_spk_conditioned_encoder': True, 'use_noise_scaled_mas': True, 'use_mel_posterior_encoder': False, 'use_duration_discriminator': True, 'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'n_layers_trans_flow': 3, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 8, 2, 2], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256}, 'num_languages': 8, 'num_tones': 16, 'symbols': ['_', '"', '(', ')', '*', '/', ':', 'AA', 'E', 'EE', 'En', 'N', 'OO', 'Q', 'V', '[', '\\', ']', '^', 'a', 'a:', 'aa', 'ae', 'ah', 'ai', 'an', 'ang', 'ao', 'aw', 'ay', 'b', 'by', 'c', 'ch', 'd', 'dh', 'dy', 'e', 'e:', 'eh', 'ei', 'en', 'eng', 'er', 'ey', 'f', 'g', 'gy', 'h', 'hh', 'hy', 'i', 'i0', 'i:', 'ia', 'ian', 'iang', 'iao', 'ie', 'ih', 'in', 'ing', 'iong', 'ir', 'iu', 'iy', 'j', 'jh', 'k', 'ky', 'l', 'm', 'my', 'n', 'ng', 'ny', 'o', 'o:', 'ong', 'ou', 'ow', 'oy', 'p', 'py', 'q', 'r', 'ry', 's', 'sh', 't', 'th', 'ts', 'ty', 'u', 'u:', 'ua', 'uai', 'uan', 'uang', 'uh', 'ui', 'un', 'uo', 'uw', 'v', 'van', 've', 'vn', 'w', 'x', 'y', 'z', 'zh', 'zy', '~', 'æ', 'ç', 'ð', 'ø', 'ŋ', 'œ', 'ɐ', 'ɑ', 'ɒ', 'ɔ', 'ɕ', 'ə', 'ɛ', 'ɜ', 'ɡ', 'ɣ', 'ɥ', 'ɦ', 'ɪ', 'ɫ', 'ɬ', 'ɭ', 'ɯ', 'ɲ', 'ɵ', 'ɸ', 'ɹ', 'ɾ', 'ʁ', 'ʃ', 'ʊ', 'ʌ', 'ʎ', 'ʏ', 'ʑ', 'ʒ', 'ʝ', 'ʲ', 'ˈ', 'ˌ', 'ː', '̃', '̩', 'β', 'θ', 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆫ', 'ᆮ', 'ᆯ', 'ᆷ', 'ᆸ', 'ᆼ', 'ㄸ', '!', '?', '…', ',', '.', "'", '-', '¿', '¡', 'SP', 'UNK'], 'model_dir': './logs/example', 'pretrain_G': None, 'pretrain_D': None, 'pretrain_dur': None, 'port': 10000} |
|
2024-11-28 06:50:27,013 example WARNING /ephemeral/vyvo_ai/MeloTTS/melo is not a git repository, therefore hash value comparison will be ignored. |
|
2024-11-28 06:50:33,036 example ERROR enc_p.language_emb.weight is not in the checkpoint |
|
2024-11-28 06:50:33,036 example ERROR emb_g.weight is not in the checkpoint |
|
2024-11-28 06:50:33,069 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/73ad3d5a37c82356ed81630b0a435b4b376ca49523854fe2b8302609fd71c193.133b77b9d9162e348486a0a0778fa47d726930e3ec12ea5e2684c0c919743a65' (iteration 0) |
|
2024-11-28 06:50:33,185 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c3d3c787a8711093a79ee95f091a35de75e527b6e8e28424ad7010f6e86cce58.e5f88bb1eca17c37beb511b15a932e84fdc8b66d8a8d5c5075334650425954f2' (iteration 0) |
|
2024-11-28 06:50:33,270 example INFO Loaded checkpoint '/home/ubuntu/.cache/cached_path/c7b373ab8939eb672a985a802d21420534ca0cd43fa4aecf4fa6088a569ee2a1.ce7a8153914d9727ebc28e4b4e3d31eed35aa0b4e3d125eb54e6f8363968dd7a' (iteration 0) |
|
2024-11-28 06:50:33,273 example INFO Loaded checkpoint './logs/example/DUR_79000.pth' (iteration 191) |
|
2024-11-28 06:50:33,655 example INFO Loaded checkpoint './logs/example/G_79000.pth' (iteration 191) |
|
2024-11-28 06:50:33,782 example INFO Loaded checkpoint './logs/example/D_79000.pth' (iteration 191) |
|
2024-11-28 07:06:28,649 example INFO Train Epoch: 191 [36%] |
|
2024-11-28 07:06:28,649 example INFO [2.1360690593719482, 2.8026480674743652, 9.037551879882812, 17.236557006835938, 1.9379674196243286, 2.0023531913757324, 79000, 0.0002999625] |
|
2024-11-28 07:06:32,842 example INFO Saving model and optimizer state at iteration 191 to ./logs/example/G_79000.pth |
|
2024-11-28 07:06:34,012 example INFO Saving model and optimizer state at iteration 191 to ./logs/example/D_79000.pth |
|
2024-11-28 07:06:34,744 example INFO Saving model and optimizer state at iteration 191 to ./logs/example/DUR_79000.pth |
|
|