Hugging Face's logo

lqlklu
/
so-vits-svc-4.0-danxiao

Transformers
TensorBoard
Inference Endpoints
Model card Files Metrics Community
lqlklu commited on
Commit
abba9ee
•
1 Parent(s): ad06fe2

add files

Browse files
Files changed (8) hide show
  1. D_10600.pth +3 -0
  2. D_11500.pth +3 -0
  3. G_10600.pth +3 -0
  4. G_11500.pth +3 -0
  5. config.json +94 -0
  6. eval/events.out.tfevents.1683120118.PY4ddV.774.1 +3 -0
  7. events.out.tfevents.1683120118.PY4ddV.774.0 +3 -0
  8. train.log +852 -0
D_10600.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f95f94ba4337d8184aa6d828f5ac1e2f74346ea8c651c67d1af65133a6c069c3
3
+ size 561098185
D_11500.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cadcb0a03397ff5bf528afbc73ad31c2859f3f14886346db5f8ac3209ad5388e
3
+ size 561098185
G_10600.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6af4253cb85eb6f6bdea557098501f54942da8ba5533606ef26427ace7478f0
3
+ size 542789405
G_11500.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4defb1685aadaf969dd438efe55acd168c61f1db485b77981099023fa02cdf5
3
+ size 542789405
config.json ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "log_interval": 100,
4
+ "eval_interval": 100,
5
+ "seed": 1234,
6
+ "epochs": 800,
7
+ "learning_rate": 0.0001,
8
+ "betas": [
9
+ 0.8,
10
+ 0.99
11
+ ],
12
+ "eps": 1e-09,
13
+ "batch_size": 6,
14
+ "fp16_run": false,
15
+ "lr_decay": 0.999875,
16
+ "segment_size": 10240,
17
+ "init_lr_ratio": 1,
18
+ "warmup_epochs": 0,
19
+ "c_mel": 45,
20
+ "c_kl": 1.0,
21
+ "use_sr": true,
22
+ "max_speclen": 512,
23
+ "port": "8001",
24
+ "keep_ckpts": 2,
25
+ "all_in_mem": false
26
+ },
27
+ "data": {
28
+ "training_files": "filelists/train.txt",
29
+ "validation_files": "filelists/val.txt",
30
+ "max_wav_value": 32768.0,
31
+ "sampling_rate": 44100,
32
+ "filter_length": 2048,
33
+ "hop_length": 512,
34
+ "win_length": 2048,
35
+ "n_mel_channels": 80,
36
+ "mel_fmin": 0.0,
37
+ "mel_fmax": 22050
38
+ },
39
+ "model": {
40
+ "inter_channels": 192,
41
+ "hidden_channels": 192,
42
+ "filter_channels": 768,
43
+ "n_heads": 2,
44
+ "n_layers": 6,
45
+ "kernel_size": 3,
46
+ "p_dropout": 0.1,
47
+ "resblock": "1",
48
+ "resblock_kernel_sizes": [
49
+ 3,
50
+ 7,
51
+ 11
52
+ ],
53
+ "resblock_dilation_sizes": [
54
+ [
55
+ 1,
56
+ 3,
57
+ 5
58
+ ],
59
+ [
60
+ 1,
61
+ 3,
62
+ 5
63
+ ],
64
+ [
65
+ 1,
66
+ 3,
67
+ 5
68
+ ]
69
+ ],
70
+ "upsample_rates": [
71
+ 8,
72
+ 8,
73
+ 2,
74
+ 2,
75
+ 2
76
+ ],
77
+ "upsample_initial_channel": 512,
78
+ "upsample_kernel_sizes": [
79
+ 16,
80
+ 16,
81
+ 4,
82
+ 4,
83
+ 4
84
+ ],
85
+ "n_layers_q": 3,
86
+ "use_spectral_norm": false,
87
+ "gin_channels": 256,
88
+ "ssl_dim": 256,
89
+ "n_speakers": 200
90
+ },
91
+ "spk": {
92
+ "danxiao": 0
93
+ }
94
+ }
eval/events.out.tfevents.1683120118.PY4ddV.774.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2f938061d4f31445606faafd1737255c66e9524b5ba5bbb6faf4f1687cb10751
3
+ size 345535863
events.out.tfevents.1683120118.PY4ddV.774.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2cb2c2ffc50ce28ddb00f019a8ac4f95711b47e1784003aabb145d2282e9f6eb
3
+ size 22616873
train.log ADDED
@@ -0,0 +1,852 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2023-05-03 13:21:58,872 44k INFO {'train': {'log_interval': 100, 'eval_interval': 100, 'seed': 1234, 'epochs': 800, 'learning_rate': 0.0001, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 10240, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'use_sr': True, 'max_speclen': 512, 'port': '8001', 'keep_ckpts': 2, 'all_in_mem': False}, 'data': {'training_files': 'filelists/train.txt', 'validation_files': 'filelists/val.txt', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 80, 'mel_fmin': 0.0, 'mel_fmax': 22050}, 'model': {'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 4, 4, 4], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256, 'ssl_dim': 256, 'n_speakers': 200}, 'spk': {'danxiao': 0}, 'model_dir': './logs/44k'}
2
+ 2023-05-03 13:22:01,597 44k INFO emb_g.weight is not in the checkpoint
3
+ 2023-05-03 13:22:01,663 44k INFO Loaded checkpoint './logs/44k/G_0.pth' (iteration 0)
4
+ 2023-05-03 13:22:01,753 44k INFO Loaded checkpoint './logs/44k/D_0.pth' (iteration 0)
5
+ 2023-05-03 13:22:55,617 44k INFO ====> Epoch: 1, cost 56.75 s
6
+ 2023-05-03 13:23:16,634 44k INFO Train Epoch: 2 [39%]
7
+ 2023-05-03 13:23:16,635 44k INFO Losses: [2.0492827892303467, 2.659921646118164, 10.786680221557617, 31.86163330078125, 1.4900364875793457], step: 100, lr: 9.99875e-05, reference_loss: 48.84755325317383
8
+ 2023-05-03 13:23:23,145 44k INFO Saving model and optimizer state at iteration 2 to ./logs/44k/G_100.pth
9
+ 2023-05-03 13:23:24,867 44k INFO Saving model and optimizer state at iteration 2 to ./logs/44k/D_100.pth
10
+ 2023-05-03 13:23:50,972 44k INFO ====> Epoch: 2, cost 55.35 s
11
+ 2023-05-03 13:24:29,866 44k INFO Train Epoch: 3 [80%]
12
+ 2023-05-03 13:24:29,867 44k INFO Losses: [2.5209755897521973, 2.313697576522827, 9.055400848388672, 24.980056762695312, 1.70906400680542], step: 200, lr: 9.99750015625e-05, reference_loss: 40.57919692993164
13
+ 2023-05-03 13:24:34,955 44k INFO Saving model and optimizer state at iteration 3 to ./logs/44k/G_200.pth
14
+ 2023-05-03 13:24:36,690 44k INFO Saving model and optimizer state at iteration 3 to ./logs/44k/D_200.pth
15
+ 2023-05-03 13:24:45,486 44k INFO ====> Epoch: 3, cost 54.51 s
16
+ 2023-05-03 13:25:32,621 44k INFO ====> Epoch: 4, cost 47.14 s
17
+ 2023-05-03 13:25:46,099 44k INFO Train Epoch: 5 [21%]
18
+ 2023-05-03 13:25:46,100 44k INFO Losses: [2.311826467514038, 2.4695792198181152, 10.9570894241333, 22.569786071777344, 1.274253249168396], step: 300, lr: 9.995000937421877e-05, reference_loss: 39.58253479003906
19
+ 2023-05-03 13:25:51,325 44k INFO Saving model and optimizer state at iteration 5 to ./logs/44k/G_300.pth
20
+ 2023-05-03 13:25:53,076 44k INFO Saving model and optimizer state at iteration 5 to ./logs/44k/D_300.pth
21
+ 2023-05-03 13:25:53,605 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_100.pth
22
+ 2023-05-03 13:25:53,634 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_100.pth
23
+ 2023-05-03 13:26:27,242 44k INFO ====> Epoch: 5, cost 54.62 s
24
+ 2023-05-03 13:26:59,132 44k INFO Train Epoch: 6 [62%]
25
+ 2023-05-03 13:26:59,133 44k INFO Losses: [2.14005708694458, 2.4915013313293457, 10.79394817352295, 22.39518928527832, 1.7690892219543457], step: 400, lr: 9.993751562304699e-05, reference_loss: 39.58978271484375
26
+ 2023-05-03 13:27:04,372 44k INFO Saving model and optimizer state at iteration 6 to ./logs/44k/G_400.pth
27
+ 2023-05-03 13:27:06,083 44k INFO Saving model and optimizer state at iteration 6 to ./logs/44k/D_400.pth
28
+ 2023-05-03 13:27:06,613 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_200.pth
29
+ 2023-05-03 13:27:06,642 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_200.pth
30
+ 2023-05-03 13:27:22,649 44k INFO ====> Epoch: 6, cost 55.41 s
31
+ 2023-05-03 13:28:09,748 44k INFO ====> Epoch: 7, cost 47.10 s
32
+ 2023-05-03 13:28:15,219 44k INFO Train Epoch: 8 [3%]
33
+ 2023-05-03 13:28:15,219 44k INFO Losses: [2.4994237422943115, 2.2374916076660156, 10.99351692199707, 27.435150146484375, 1.3526171445846558], step: 500, lr: 9.991253280566489e-05, reference_loss: 44.51819610595703
34
+ 2023-05-03 13:28:20,329 44k INFO Saving model and optimizer state at iteration 8 to ./logs/44k/G_500.pth
35
+ 2023-05-03 13:28:22,212 44k INFO Saving model and optimizer state at iteration 8 to ./logs/44k/D_500.pth
36
+ 2023-05-03 13:28:22,746 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_300.pth
37
+ 2023-05-03 13:28:22,774 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_300.pth
38
+ 2023-05-03 13:29:04,358 44k INFO ====> Epoch: 8, cost 54.61 s
39
+ 2023-05-03 13:29:27,578 44k INFO Train Epoch: 9 [44%]
40
+ 2023-05-03 13:29:27,579 44k INFO Losses: [2.4906890392303467, 2.554823637008667, 9.332549095153809, 23.424272537231445, 1.4919406175613403], step: 600, lr: 9.990004373906418e-05, reference_loss: 39.294273376464844
41
+ 2023-05-03 13:29:32,778 44k INFO Saving model and optimizer state at iteration 9 to ./logs/44k/G_600.pth
42
+ 2023-05-03 13:29:34,547 44k INFO Saving model and optimizer state at iteration 9 to ./logs/44k/D_600.pth
43
+ 2023-05-03 13:29:35,074 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_400.pth
44
+ 2023-05-03 13:29:35,102 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_400.pth
45
+ 2023-05-03 13:29:59,010 44k INFO ====> Epoch: 9, cost 54.65 s
46
+ 2023-05-03 13:30:39,940 44k INFO Train Epoch: 10 [85%]
47
+ 2023-05-03 13:30:39,941 44k INFO Losses: [2.4156060218811035, 2.1082849502563477, 11.128122329711914, 24.051517486572266, 1.5151060819625854], step: 700, lr: 9.98875562335968e-05, reference_loss: 41.21863555908203
48
+ 2023-05-03 13:30:45,107 44k INFO Saving model and optimizer state at iteration 10 to ./logs/44k/G_700.pth
49
+ 2023-05-03 13:30:46,837 44k INFO Saving model and optimizer state at iteration 10 to ./logs/44k/D_700.pth
50
+ 2023-05-03 13:30:47,368 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_500.pth
51
+ 2023-05-03 13:30:47,397 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_500.pth
52
+ 2023-05-03 13:30:53,795 44k INFO ====> Epoch: 10, cost 54.79 s
53
+ 2023-05-03 13:31:40,858 44k INFO ====> Epoch: 11, cost 47.06 s
54
+ 2023-05-03 13:31:57,432 44k INFO Train Epoch: 12 [25%]
55
+ 2023-05-03 13:31:57,433 44k INFO Losses: [2.3472788333892822, 2.697023630142212, 9.64306354522705, 21.859516143798828, 1.6248269081115723], step: 800, lr: 9.986258590528146e-05, reference_loss: 38.17171096801758
56
+ 2023-05-03 13:32:02,662 44k INFO Saving model and optimizer state at iteration 12 to ./logs/44k/G_800.pth
57
+ 2023-05-03 13:32:04,334 44k INFO Saving model and optimizer state at iteration 12 to ./logs/44k/D_800.pth
58
+ 2023-05-03 13:32:04,862 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_600.pth
59
+ 2023-05-03 13:32:04,890 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_600.pth
60
+ 2023-05-03 13:32:36,561 44k INFO ====> Epoch: 12, cost 55.70 s
61
+ 2023-05-03 13:33:09,525 44k INFO Train Epoch: 13 [66%]
62
+ 2023-05-03 13:33:09,526 44k INFO Losses: [2.51536226272583, 2.6379146575927734, 7.380555629730225, 22.201026916503906, 1.308340072631836], step: 900, lr: 9.98501030820433e-05, reference_loss: 36.04319763183594
63
+ 2023-05-03 13:33:14,629 44k INFO Saving model and optimizer state at iteration 13 to ./logs/44k/G_900.pth
64
+ 2023-05-03 13:33:16,494 44k INFO Saving model and optimizer state at iteration 13 to ./logs/44k/D_900.pth
65
+ 2023-05-03 13:33:17,022 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_700.pth
66
+ 2023-05-03 13:33:17,051 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_700.pth
67
+ 2023-05-03 13:33:31,182 44k INFO ====> Epoch: 13, cost 54.62 s
68
+ 2023-05-03 13:34:17,814 44k INFO ====> Epoch: 14, cost 46.63 s
69
+ 2023-05-03 13:34:25,110 44k INFO Train Epoch: 15 [7%]
70
+ 2023-05-03 13:34:25,111 44k INFO Losses: [2.690953493118286, 2.0789146423339844, 8.32895278930664, 21.45608139038086, 1.2996363639831543], step: 1000, lr: 9.982514211643064e-05, reference_loss: 35.85453796386719
71
+ 2023-05-03 13:34:30,133 44k INFO Saving model and optimizer state at iteration 15 to ./logs/44k/G_1000.pth
72
+ 2023-05-03 13:34:31,795 44k INFO Saving model and optimizer state at iteration 15 to ./logs/44k/D_1000.pth
73
+ 2023-05-03 13:34:32,325 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_800.pth
74
+ 2023-05-03 13:34:32,353 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_800.pth
75
+ 2023-05-03 13:35:11,691 44k INFO ====> Epoch: 15, cost 53.88 s
76
+ 2023-05-03 13:35:39,789 44k INFO Train Epoch: 16 [48%]
77
+ 2023-05-03 13:35:39,790 44k INFO Losses: [2.3972558975219727, 2.426987409591675, 8.578001022338867, 24.267925262451172, 1.3725520372390747], step: 1100, lr: 9.981266397366609e-05, reference_loss: 39.042720794677734
78
+ 2023-05-03 13:35:44,968 44k INFO Saving model and optimizer state at iteration 16 to ./logs/44k/G_1100.pth
79
+ 2023-05-03 13:35:46,747 44k INFO Saving model and optimizer state at iteration 16 to ./logs/44k/D_1100.pth
80
+ 2023-05-03 13:35:47,285 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_900.pth
81
+ 2023-05-03 13:35:47,313 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_900.pth
82
+ 2023-05-03 13:36:10,496 44k INFO ====> Epoch: 16, cost 58.81 s
83
+ 2023-05-03 13:36:57,589 44k INFO Train Epoch: 17 [89%]
84
+ 2023-05-03 13:36:57,590 44k INFO Losses: [2.3657584190368652, 2.3757505416870117, 8.66689395904541, 22.137908935546875, 1.4960813522338867], step: 1200, lr: 9.980018739066937e-05, reference_loss: 37.04239273071289
85
+ 2023-05-03 13:37:02,822 44k INFO Saving model and optimizer state at iteration 17 to ./logs/44k/G_1200.pth
86
+ 2023-05-03 13:37:04,581 44k INFO Saving model and optimizer state at iteration 17 to ./logs/44k/D_1200.pth
87
+ 2023-05-03 13:37:05,121 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1000.pth
88
+ 2023-05-03 13:37:05,150 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1000.pth
89
+ 2023-05-03 13:37:10,222 44k INFO ====> Epoch: 17, cost 59.73 s
90
+ 2023-05-03 13:37:56,991 44k INFO ====> Epoch: 18, cost 46.77 s
91
+ 2023-05-03 13:38:13,880 44k INFO Train Epoch: 19 [30%]
92
+ 2023-05-03 13:38:13,881 44k INFO Losses: [2.3961024284362793, 2.4425392150878906, 10.046466827392578, 23.261734008789062, 1.3326858282089233], step: 1300, lr: 9.977523890319963e-05, reference_loss: 39.479530334472656
93
+ 2023-05-03 13:38:18,930 44k INFO Saving model and optimizer state at iteration 19 to ./logs/44k/G_1300.pth
94
+ 2023-05-03 13:38:20,769 44k INFO Saving model and optimizer state at iteration 19 to ./logs/44k/D_1300.pth
95
+ 2023-05-03 13:38:21,295 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1100.pth
96
+ 2023-05-03 13:38:21,323 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1100.pth
97
+ 2023-05-03 13:38:51,063 44k INFO ====> Epoch: 19, cost 54.07 s
98
+ 2023-05-03 13:39:26,290 44k INFO Train Epoch: 20 [70%]
99
+ 2023-05-03 13:39:26,290 44k INFO Losses: [2.4467904567718506, 2.1601767539978027, 9.989028930664062, 21.93836784362793, 1.6368579864501953], step: 1400, lr: 9.976276699833672e-05, reference_loss: 38.17121887207031
100
+ 2023-05-03 13:39:31,241 44k INFO Saving model and optimizer state at iteration 20 to ./logs/44k/G_1400.pth
101
+ 2023-05-03 13:39:32,988 44k INFO Saving model and optimizer state at iteration 20 to ./logs/44k/D_1400.pth
102
+ 2023-05-03 13:39:33,541 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1200.pth
103
+ 2023-05-03 13:39:33,570 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1200.pth
104
+ 2023-05-03 13:39:46,142 44k INFO ====> Epoch: 20, cost 55.08 s
105
+ 2023-05-03 13:40:33,445 44k INFO ====> Epoch: 21, cost 47.30 s
106
+ 2023-05-03 13:40:42,585 44k INFO Train Epoch: 22 [11%]
107
+ 2023-05-03 13:40:42,586 44k INFO Losses: [2.4825923442840576, 1.9403694868087769, 9.403828620910645, 24.17758560180664, 1.701635718345642], step: 1500, lr: 9.973782786538036e-05, reference_loss: 39.70601272583008
108
+ 2023-05-03 13:40:47,668 44k INFO Saving model and optimizer state at iteration 22 to ./logs/44k/G_1500.pth
109
+ 2023-05-03 13:40:49,405 44k INFO Saving model and optimizer state at iteration 22 to ./logs/44k/D_1500.pth
110
+ 2023-05-03 13:40:49,943 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1300.pth
111
+ 2023-05-03 13:40:49,972 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1300.pth
112
+ 2023-05-03 13:41:27,922 44k INFO ====> Epoch: 22, cost 54.48 s
113
+ 2023-05-03 13:41:54,792 44k INFO Train Epoch: 23 [52%]
114
+ 2023-05-03 13:41:54,793 44k INFO Losses: [2.400172472000122, 2.8658950328826904, 10.408707618713379, 25.947738647460938, 0.9147400856018066], step: 1600, lr: 9.972536063689719e-05, reference_loss: 42.537254333496094
115
+ 2023-05-03 13:42:00,055 44k INFO Saving model and optimizer state at iteration 23 to ./logs/44k/G_1600.pth
116
+ 2023-05-03 13:42:01,769 44k INFO Saving model and optimizer state at iteration 23 to ./logs/44k/D_1600.pth
117
+ 2023-05-03 13:42:02,304 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1400.pth
118
+ 2023-05-03 13:42:02,332 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1400.pth
119
+ 2023-05-03 13:42:22,651 44k INFO ====> Epoch: 23, cost 54.73 s
120
+ 2023-05-03 13:43:11,447 44k INFO Train Epoch: 24 [93%]
121
+ 2023-05-03 13:43:11,448 44k INFO Losses: [2.320039749145508, 2.4628827571868896, 8.813612937927246, 21.163532257080078, 1.414304494857788], step: 1700, lr: 9.971289496681757e-05, reference_loss: 36.17436981201172
122
+ 2023-05-03 13:43:16,627 44k INFO Saving model and optimizer state at iteration 24 to ./logs/44k/G_1700.pth
123
+ 2023-05-03 13:43:18,476 44k INFO Saving model and optimizer state at iteration 24 to ./logs/44k/D_1700.pth
124
+ 2023-05-03 13:43:19,015 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1500.pth
125
+ 2023-05-03 13:43:19,043 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1500.pth
126
+ 2023-05-03 13:43:22,120 44k INFO ====> Epoch: 24, cost 59.47 s
127
+ 2023-05-03 13:44:08,810 44k INFO ====> Epoch: 25, cost 46.69 s
128
+ 2023-05-03 13:44:28,358 44k INFO Train Epoch: 26 [34%]
129
+ 2023-05-03 13:44:28,359 44k INFO Losses: [2.3216712474823, 2.4774258136749268, 9.157946586608887, 20.46706199645996, 1.4309511184692383], step: 1800, lr: 9.968796830108985e-05, reference_loss: 35.85505676269531
130
+ 2023-05-03 13:44:33,579 44k INFO Saving model and optimizer state at iteration 26 to ./logs/44k/G_1800.pth
131
+ 2023-05-03 13:44:35,263 44k INFO Saving model and optimizer state at iteration 26 to ./logs/44k/D_1800.pth
132
+ 2023-05-03 13:44:35,805 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1600.pth
133
+ 2023-05-03 13:44:35,834 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1600.pth
134
+ 2023-05-03 13:45:07,327 44k INFO ====> Epoch: 26, cost 58.52 s
135
+ 2023-05-03 13:45:43,605 44k INFO Train Epoch: 27 [75%]
136
+ 2023-05-03 13:45:43,606 44k INFO Losses: [2.493234395980835, 2.7097132205963135, 7.805803298950195, 22.947893142700195, 1.7040966749191284], step: 1900, lr: 9.967550730505221e-05, reference_loss: 37.66073989868164
137
+ 2023-05-03 13:45:48,718 44k INFO Saving model and optimizer state at iteration 27 to ./logs/44k/G_1900.pth
138
+ 2023-05-03 13:45:50,379 44k INFO Saving model and optimizer state at iteration 27 to ./logs/44k/D_1900.pth
139
+ 2023-05-03 13:45:50,920 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1700.pth
140
+ 2023-05-03 13:45:50,949 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1700.pth
141
+ 2023-05-03 13:46:01,578 44k INFO ====> Epoch: 27, cost 54.25 s
142
+ 2023-05-03 13:46:48,254 44k INFO ====> Epoch: 28, cost 46.68 s
143
+ 2023-05-03 13:46:59,224 44k INFO Train Epoch: 29 [15%]
144
+ 2023-05-03 13:46:59,225 44k INFO Losses: [2.339369773864746, 2.4442389011383057, 11.337064743041992, 22.017444610595703, 1.5041507482528687], step: 2000, lr: 9.965058998565574e-05, reference_loss: 39.642269134521484
145
+ 2023-05-03 13:47:04,425 44k INFO Saving model and optimizer state at iteration 29 to ./logs/44k/G_2000.pth
146
+ 2023-05-03 13:47:06,082 44k INFO Saving model and optimizer state at iteration 29 to ./logs/44k/D_2000.pth
147
+ 2023-05-03 13:47:06,617 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1800.pth
148
+ 2023-05-03 13:47:06,646 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1800.pth
149
+ 2023-05-03 13:47:42,811 44k INFO ====> Epoch: 29, cost 54.56 s
150
+ 2023-05-03 13:48:11,315 44k INFO Train Epoch: 30 [56%]
151
+ 2023-05-03 13:48:11,316 44k INFO Losses: [2.388498306274414, 2.5379557609558105, 7.413846015930176, 23.07448959350586, 1.256014108657837], step: 2100, lr: 9.963813366190753e-05, reference_loss: 36.67080307006836
152
+ 2023-05-03 13:48:16,439 44k INFO Saving model and optimizer state at iteration 30 to ./logs/44k/G_2100.pth
153
+ 2023-05-03 13:48:18,342 44k INFO Saving model and optimizer state at iteration 30 to ./logs/44k/D_2100.pth
154
+ 2023-05-03 13:48:18,874 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1900.pth
155
+ 2023-05-03 13:48:18,903 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1900.pth
156
+ 2023-05-03 13:48:37,697 44k INFO ====> Epoch: 30, cost 54.89 s
157
+ 2023-05-03 13:49:23,424 44k INFO Train Epoch: 31 [97%]
158
+ 2023-05-03 13:49:23,424 44k INFO Losses: [2.6424458026885986, 2.4859418869018555, 9.348760604858398, 21.602563858032227, 1.1841977834701538], step: 2200, lr: 9.962567889519979e-05, reference_loss: 37.26390838623047
159
+ 2023-05-03 13:49:28,489 44k INFO Saving model and optimizer state at iteration 31 to ./logs/44k/G_2200.pth
160
+ 2023-05-03 13:49:30,184 44k INFO Saving model and optimizer state at iteration 31 to ./logs/44k/D_2200.pth
161
+ 2023-05-03 13:49:30,714 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2000.pth
162
+ 2023-05-03 13:49:30,743 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2000.pth
163
+ 2023-05-03 13:49:31,809 44k INFO ====> Epoch: 31, cost 54.11 s
164
+ 2023-05-03 13:50:18,662 44k INFO ====> Epoch: 32, cost 46.85 s
165
+ 2023-05-03 13:50:39,138 44k INFO Train Epoch: 33 [38%]
166
+ 2023-05-03 13:50:39,139 44k INFO Losses: [2.171383857727051, 2.7099664211273193, 9.955101013183594, 30.65108299255371, 1.07896089553833], step: 2300, lr: 9.960077403212722e-05, reference_loss: 46.56649398803711
167
+ 2023-05-03 13:50:44,292 44k INFO Saving model and optimizer state at iteration 33 to ./logs/44k/G_2300.pth
168
+ 2023-05-03 13:50:45,922 44k INFO Saving model and optimizer state at iteration 33 to ./logs/44k/D_2300.pth
169
+ 2023-05-03 13:50:46,454 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2100.pth
170
+ 2023-05-03 13:50:46,483 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2100.pth
171
+ 2023-05-03 13:51:12,651 44k INFO ====> Epoch: 33, cost 53.99 s
172
+ 2023-05-03 13:51:51,078 44k INFO Train Epoch: 34 [79%]
173
+ 2023-05-03 13:51:51,079 44k INFO Losses: [2.7176640033721924, 2.151402473449707, 7.5227837562561035, 22.640493392944336, 1.2028143405914307], step: 2400, lr: 9.95883239353732e-05, reference_loss: 36.23515701293945
174
+ 2023-05-03 13:51:56,385 44k INFO Saving model and optimizer state at iteration 34 to ./logs/44k/G_2400.pth
175
+ 2023-05-03 13:51:58,063 44k INFO Saving model and optimizer state at iteration 34 to ./logs/44k/D_2400.pth
176
+ 2023-05-03 13:51:58,598 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2200.pth
177
+ 2023-05-03 13:51:58,627 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2200.pth
178
+ 2023-05-03 13:52:07,487 44k INFO ====> Epoch: 34, cost 54.84 s
179
+ 2023-05-03 13:52:54,709 44k INFO ====> Epoch: 35, cost 47.22 s
180
+ 2023-05-03 13:53:07,508 44k INFO Train Epoch: 36 [20%]
181
+ 2023-05-03 13:53:07,508 44k INFO Losses: [2.1929237842559814, 2.455925941467285, 8.182902336120605, 22.028627395629883, 1.0674852132797241], step: 2500, lr: 9.956342841045691e-05, reference_loss: 35.92786407470703
182
+ 2023-05-03 13:53:12,777 44k INFO Saving model and optimizer state at iteration 36 to ./logs/44k/G_2500.pth
183
+ 2023-05-03 13:53:14,484 44k INFO Saving model and optimizer state at iteration 36 to ./logs/44k/D_2500.pth
184
+ 2023-05-03 13:53:15,023 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2300.pth
185
+ 2023-05-03 13:53:15,052 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2300.pth
186
+ 2023-05-03 13:53:50,858 44k INFO ====> Epoch: 36, cost 56.15 s
187
+ 2023-05-03 13:54:21,588 44k INFO Train Epoch: 37 [61%]
188
+ 2023-05-03 13:54:21,589 44k INFO Losses: [2.3986458778381348, 2.24863338470459, 10.338911056518555, 22.920942306518555, 1.7659378051757812], step: 2600, lr: 9.95509829819056e-05, reference_loss: 39.673072814941406
189
+ 2023-05-03 13:54:26,751 44k INFO Saving model and optimizer state at iteration 37 to ./logs/44k/G_2600.pth
190
+ 2023-05-03 13:54:28,569 44k INFO Saving model and optimizer state at iteration 37 to ./logs/44k/D_2600.pth
191
+ 2023-05-03 13:54:29,104 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2400.pth
192
+ 2023-05-03 13:54:29,132 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2400.pth
193
+ 2023-05-03 13:54:45,792 44k INFO ====> Epoch: 37, cost 54.93 s
194
+ 2023-05-03 13:55:32,897 44k INFO ====> Epoch: 38, cost 47.11 s
195
+ 2023-05-03 13:55:37,793 44k INFO Train Epoch: 39 [1%]
196
+ 2023-05-03 13:55:37,793 44k INFO Losses: [2.5867271423339844, 2.1795637607574463, 8.613794326782227, 22.480947494506836, 1.3132673501968384], step: 2700, lr: 9.952609679164422e-05, reference_loss: 37.17429733276367
197
+ 2023-05-03 13:55:42,846 44k INFO Saving model and optimizer state at iteration 39 to ./logs/44k/G_2700.pth
198
+ 2023-05-03 13:55:44,488 44k INFO Saving model and optimizer state at iteration 39 to ./logs/44k/D_2700.pth
199
+ 2023-05-03 13:55:45,015 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2500.pth
200
+ 2023-05-03 13:55:45,043 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2500.pth
201
+ 2023-05-03 13:56:26,924 44k INFO ====> Epoch: 39, cost 54.03 s
202
+ 2023-05-03 13:56:49,364 44k INFO Train Epoch: 40 [42%]
203
+ 2023-05-03 13:56:49,364 44k INFO Losses: [2.073730945587158, 2.529214382171631, 10.178397178649902, 23.654401779174805, 1.5418543815612793], step: 2800, lr: 9.951365602954526e-05, reference_loss: 39.97760009765625
204
+ 2023-05-03 13:56:54,566 44k INFO Saving model and optimizer state at iteration 40 to ./logs/44k/G_2800.pth
205
+ 2023-05-03 13:56:56,221 44k INFO Saving model and optimizer state at iteration 40 to ./logs/44k/D_2800.pth
206
+ 2023-05-03 13:56:56,751 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2600.pth
207
+ 2023-05-03 13:56:56,779 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2600.pth
208
+ 2023-05-03 13:57:21,240 44k INFO ====> Epoch: 40, cost 54.32 s
209
+ 2023-05-03 13:58:01,611 44k INFO Train Epoch: 41 [83%]
210
+ 2023-05-03 13:58:01,612 44k INFO Losses: [2.2060742378234863, 2.293834686279297, 9.187530517578125, 23.000995635986328, 1.1135773658752441], step: 2900, lr: 9.950121682254156e-05, reference_loss: 37.8020133972168
211
+ 2023-05-03 13:58:06,827 44k INFO Saving model and optimizer state at iteration 41 to ./logs/44k/G_2900.pth
212
+ 2023-05-03 13:58:08,418 44k INFO Saving model and optimizer state at iteration 41 to ./logs/44k/D_2900.pth
213
+ 2023-05-03 13:58:08,952 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2700.pth
214
+ 2023-05-03 13:58:08,981 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2700.pth
215
+ 2023-05-03 13:58:16,038 44k INFO ====> Epoch: 41, cost 54.80 s
216
+ 2023-05-03 13:59:03,415 44k INFO ====> Epoch: 42, cost 47.38 s
217
+ 2023-05-03 13:59:17,934 44k INFO Train Epoch: 43 [24%]
218
+ 2023-05-03 13:59:17,935 44k INFO Losses: [2.6567223072052, 2.23923921585083, 8.49673080444336, 21.460689544677734, 1.3385343551635742], step: 3000, lr: 9.947634307304244e-05, reference_loss: 36.191917419433594
219
+ 2023-05-03 13:59:23,051 44k INFO Saving model and optimizer state at iteration 43 to ./logs/44k/G_3000.pth
220
+ 2023-05-03 13:59:24,847 44k INFO Saving model and optimizer state at iteration 43 to ./logs/44k/D_3000.pth
221
+ 2023-05-03 13:59:25,379 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2800.pth
222
+ 2023-05-03 13:59:25,407 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2800.pth
223
+ 2023-05-03 13:59:57,447 44k INFO ====> Epoch: 43, cost 54.03 s
224
+ 2023-05-03 14:00:29,515 44k INFO Train Epoch: 44 [65%]
225
+ 2023-05-03 14:00:29,516 44k INFO Losses: [2.4680213928222656, 2.120514154434204, 9.469073295593262, 20.154348373413086, 0.9433535933494568], step: 3100, lr: 9.94639085301583e-05, reference_loss: 35.15530776977539
226
+ 2023-05-03 14:00:34,549 44k INFO Saving model and optimizer state at iteration 44 to ./logs/44k/G_3100.pth
227
+ 2023-05-03 14:00:36,255 44k INFO Saving model and optimizer state at iteration 44 to ./logs/44k/D_3100.pth
228
+ 2023-05-03 14:00:36,789 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2900.pth
229
+ 2023-05-03 14:00:36,817 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2900.pth
230
+ 2023-05-03 14:00:51,446 44k INFO ====> Epoch: 44, cost 54.00 s
231
+ 2023-05-03 14:01:38,351 44k INFO ====> Epoch: 45, cost 46.91 s
232
+ 2023-05-03 14:01:45,125 44k INFO Train Epoch: 46 [6%]
233
+ 2023-05-03 14:01:45,126 44k INFO Losses: [2.278339385986328, 2.6492905616760254, 11.130102157592773, 22.5175838470459, 1.442008137702942], step: 3200, lr: 9.943904410714931e-05, reference_loss: 40.01732635498047
234
+ 2023-05-03 14:01:50,356 44k INFO Saving model and optimizer state at iteration 46 to ./logs/44k/G_3200.pth
235
+ 2023-05-03 14:01:51,996 44k INFO Saving model and optimizer state at iteration 46 to ./logs/44k/D_3200.pth
236
+ 2023-05-03 14:01:52,543 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3000.pth
237
+ 2023-05-03 14:01:52,571 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3000.pth
238
+ 2023-05-03 14:02:32,921 44k INFO ====> Epoch: 46, cost 54.57 s
239
+ 2023-05-03 14:02:57,250 44k INFO Train Epoch: 47 [46%]
240
+ 2023-05-03 14:02:57,251 44k INFO Losses: [2.453352212905884, 2.0893375873565674, 9.351663589477539, 20.12795639038086, 1.1927286386489868], step: 3300, lr: 9.942661422663591e-05, reference_loss: 35.21503829956055
241
+ 2023-05-03 14:03:02,367 44k INFO Saving model and optimizer state at iteration 47 to ./logs/44k/G_3300.pth
242
+ 2023-05-03 14:03:04,113 44k INFO Saving model and optimizer state at iteration 47 to ./logs/44k/D_3300.pth
243
+ 2023-05-03 14:03:04,642 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3100.pth
244
+ 2023-05-03 14:03:04,671 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3100.pth
245
+ 2023-05-03 14:03:27,420 44k INFO ====> Epoch: 47, cost 54.50 s
246
+ 2023-05-03 14:04:09,331 44k INFO Train Epoch: 48 [87%]
247
+ 2023-05-03 14:04:09,332 44k INFO Losses: [2.1655383110046387, 2.269059896469116, 12.266657829284668, 22.0493106842041, 0.909777045249939], step: 3400, lr: 9.941418589985758e-05, reference_loss: 39.660343170166016
248
+ 2023-05-03 14:04:14,409 44k INFO Saving model and optimizer state at iteration 48 to ./logs/44k/G_3400.pth
249
+ 2023-05-03 14:04:15,991 44k INFO Saving model and optimizer state at iteration 48 to ./logs/44k/D_3400.pth
250
+ 2023-05-03 14:04:16,523 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3200.pth
251
+ 2023-05-03 14:04:16,552 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3200.pth
252
+ 2023-05-03 14:04:21,824 44k INFO ====> Epoch: 48, cost 54.40 s
253
+ 2023-05-03 14:05:08,501 44k INFO ====> Epoch: 49, cost 46.68 s
254
+ 2023-05-03 14:05:25,017 44k INFO Train Epoch: 50 [28%]
255
+ 2023-05-03 14:05:25,018 44k INFO Losses: [2.5194897651672363, 2.653073310852051, 9.11046028137207, 19.312061309814453, 1.3811150789260864], step: 3500, lr: 9.938933390672926e-05, reference_loss: 34.976200103759766
256
+ 2023-05-03 14:05:30,215 44k INFO Saving model and optimizer state at iteration 50 to ./logs/44k/G_3500.pth
257
+ 2023-05-03 14:05:31,888 44k INFO Saving model and optimizer state at iteration 50 to ./logs/44k/D_3500.pth
258
+ 2023-05-03 14:05:32,422 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3300.pth
259
+ 2023-05-03 14:05:32,451 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3300.pth
260
+ 2023-05-03 14:06:03,174 44k INFO ====> Epoch: 50, cost 54.67 s
261
+ 2023-05-03 14:06:40,848 44k INFO Train Epoch: 51 [69%]
262
+ 2023-05-03 14:06:40,849 44k INFO Losses: [2.2235589027404785, 2.4128994941711426, 9.080496788024902, 21.989484786987305, 1.191401720046997], step: 3600, lr: 9.937691023999092e-05, reference_loss: 36.89784240722656
263
+ 2023-05-03 14:06:46,122 44k INFO Saving model and optimizer state at iteration 51 to ./logs/44k/G_3600.pth
264
+ 2023-05-03 14:06:47,781 44k INFO Saving model and optimizer state at iteration 51 to ./logs/44k/D_3600.pth
265
+ 2023-05-03 14:06:48,316 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3400.pth
266
+ 2023-05-03 14:06:48,345 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3400.pth
267
+ 2023-05-03 14:07:01,367 44k INFO ====> Epoch: 51, cost 58.19 s
268
+ 2023-05-03 14:07:54,030 44k INFO ====> Epoch: 52, cost 52.66 s
269
+ 2023-05-03 14:08:02,552 44k INFO Train Epoch: 53 [10%]
270
+ 2023-05-03 14:08:02,554 44k INFO Losses: [2.477534770965576, 2.28418231010437, 7.059985637664795, 17.771181106567383, 1.0846796035766602], step: 3700, lr: 9.935206756519513e-05, reference_loss: 30.677562713623047
271
+ 2023-05-03 14:08:07,681 44k INFO Saving model and optimizer state at iteration 53 to ./logs/44k/G_3700.pth
272
+ 2023-05-03 14:08:09,439 44k INFO Saving model and optimizer state at iteration 53 to ./logs/44k/D_3700.pth
273
+ 2023-05-03 14:08:09,972 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3500.pth
274
+ 2023-05-03 14:08:10,000 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3500.pth
275
+ 2023-05-03 14:08:48,144 44k INFO ====> Epoch: 53, cost 54.11 s
276
+ 2023-05-03 14:09:14,429 44k INFO Train Epoch: 54 [51%]
277
+ 2023-05-03 14:09:14,430 44k INFO Losses: [2.4032552242279053, 2.1533122062683105, 9.136382102966309, 19.882036209106445, 1.3597607612609863], step: 3800, lr: 9.933964855674948e-05, reference_loss: 34.93474578857422
278
+ 2023-05-03 14:09:19,621 44k INFO Saving model and optimizer state at iteration 54 to ./logs/44k/G_3800.pth
279
+ 2023-05-03 14:09:21,455 44k INFO Saving model and optimizer state at iteration 54 to ./logs/44k/D_3800.pth
280
+ 2023-05-03 14:09:22,025 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3600.pth
281
+ 2023-05-03 14:09:22,054 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3600.pth
282
+ 2023-05-03 14:09:43,013 44k INFO ====> Epoch: 54, cost 54.87 s
283
+ 2023-05-03 14:10:26,509 44k INFO Train Epoch: 55 [92%]
284
+ 2023-05-03 14:10:26,510 44k INFO Losses: [2.3103950023651123, 2.568009614944458, 9.782731056213379, 21.263418197631836, 1.306481122970581], step: 3900, lr: 9.932723110067987e-05, reference_loss: 37.23103332519531
285
+ 2023-05-03 14:10:31,549 44k INFO Saving model and optimizer state at iteration 55 to ./logs/44k/G_3900.pth
286
+ 2023-05-03 14:10:33,204 44k INFO Saving model and optimizer state at iteration 55 to ./logs/44k/D_3900.pth
287
+ 2023-05-03 14:10:33,736 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3700.pth
288
+ 2023-05-03 14:10:33,764 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3700.pth
289
+ 2023-05-03 14:10:37,205 44k INFO ====> Epoch: 55, cost 54.19 s
290
+ 2023-05-03 14:11:24,314 44k INFO ====> Epoch: 56, cost 47.11 s
291
+ 2023-05-03 14:11:42,580 44k INFO Train Epoch: 57 [32%]
292
+ 2023-05-03 14:11:42,581 44k INFO Losses: [2.491316080093384, 2.3412065505981445, 10.123672485351562, 21.33326530456543, 0.7533155083656311], step: 4000, lr: 9.930240084489267e-05, reference_loss: 37.04277420043945
293
+ 2023-05-03 14:11:47,778 44k INFO Saving model and optimizer state at iteration 57 to ./logs/44k/G_4000.pth
294
+ 2023-05-03 14:11:49,466 44k INFO Saving model and optimizer state at iteration 57 to ./logs/44k/D_4000.pth
295
+ 2023-05-03 14:11:49,997 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3800.pth
296
+ 2023-05-03 14:11:50,026 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3800.pth
297
+ 2023-05-03 14:12:18,659 44k INFO ====> Epoch: 57, cost 54.34 s
298
+ 2023-05-03 14:12:54,591 44k INFO Train Epoch: 58 [73%]
299
+ 2023-05-03 14:12:54,592 44k INFO Losses: [2.2970752716064453, 2.240769386291504, 9.480262756347656, 21.239805221557617, 1.4316742420196533], step: 4100, lr: 9.928998804478705e-05, reference_loss: 36.6895866394043
300
+ 2023-05-03 14:12:59,699 44k INFO Saving model and optimizer state at iteration 58 to ./logs/44k/G_4100.pth
301
+ 2023-05-03 14:13:01,526 44k INFO Saving model and optimizer state at iteration 58 to ./logs/44k/D_4100.pth
302
+ 2023-05-03 14:13:02,062 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3900.pth
303
+ 2023-05-03 14:13:02,090 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3900.pth
304
+ 2023-05-03 14:13:13,278 44k INFO ====> Epoch: 58, cost 54.62 s
305
+ 2023-05-03 14:13:59,954 44k INFO ====> Epoch: 59, cost 46.68 s
306
+ 2023-05-03 14:14:10,277 44k INFO Train Epoch: 60 [14%]
307
+ 2023-05-03 14:14:10,278 44k INFO Losses: [2.385362148284912, 2.30918550491333, 8.672965049743652, 19.744047164916992, 1.3143541812896729], step: 4200, lr: 9.926516709918191e-05, reference_loss: 34.4259147644043
308
+ 2023-05-03 14:14:15,357 44k INFO Saving model and optimizer state at iteration 60 to ./logs/44k/G_4200.pth
309
+ 2023-05-03 14:14:17,015 44k INFO Saving model and optimizer state at iteration 60 to ./logs/44k/D_4200.pth
310
+ 2023-05-03 14:14:17,544 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4000.pth
311
+ 2023-05-03 14:14:17,573 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4000.pth
312
+ 2023-05-03 14:14:53,940 44k INFO ====> Epoch: 60, cost 53.99 s
313
+ 2023-05-03 14:15:21,698 44k INFO Train Epoch: 61 [55%]
314
+ 2023-05-03 14:15:21,699 44k INFO Losses: [2.2300548553466797, 2.3148417472839355, 10.437084197998047, 22.084848403930664, 1.2689226865768433], step: 4300, lr: 9.92527589532945e-05, reference_loss: 38.335750579833984
315
+ 2023-05-03 14:15:26,958 44k INFO Saving model and optimizer state at iteration 61 to ./logs/44k/G_4300.pth
316
+ 2023-05-03 14:15:28,558 44k INFO Saving model and optimizer state at iteration 61 to ./logs/44k/D_4300.pth
317
+ 2023-05-03 14:15:29,093 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4100.pth
318
+ 2023-05-03 14:15:29,121 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4100.pth
319
+ 2023-05-03 14:15:48,194 44k INFO ====> Epoch: 61, cost 54.25 s
320
+ 2023-05-03 14:16:33,630 44k INFO Train Epoch: 62 [96%]
321
+ 2023-05-03 14:16:33,630 44k INFO Losses: [2.3527278900146484, 2.323845863342285, 9.502860069274902, 22.04048728942871, 0.9736533761024475], step: 4400, lr: 9.924035235842533e-05, reference_loss: 37.193572998046875
322
+ 2023-05-03 14:16:38,801 44k INFO Saving model and optimizer state at iteration 62 to ./logs/44k/G_4400.pth
323
+ 2023-05-03 14:16:40,458 44k INFO Saving model and optimizer state at iteration 62 to ./logs/44k/D_4400.pth
324
+ 2023-05-03 14:16:40,992 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4200.pth
325
+ 2023-05-03 14:16:41,021 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4200.pth
326
+ 2023-05-03 14:16:42,687 44k INFO ====> Epoch: 62, cost 54.49 s
327
+ 2023-05-03 14:17:34,840 44k INFO ====> Epoch: 63, cost 52.15 s
328
+ 2023-05-03 14:17:54,933 44k INFO Train Epoch: 64 [37%]
329
+ 2023-05-03 14:17:54,934 44k INFO Losses: [2.385552406311035, 2.1727263927459717, 9.087684631347656, 21.040884017944336, 1.0232913494110107], step: 4500, lr: 9.921554382096622e-05, reference_loss: 35.710140228271484
330
+ 2023-05-03 14:18:00,018 44k INFO Saving model and optimizer state at iteration 64 to ./logs/44k/G_4500.pth
331
+ 2023-05-03 14:18:01,734 44k INFO Saving model and optimizer state at iteration 64 to ./logs/44k/D_4500.pth
332
+ 2023-05-03 14:18:02,268 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4300.pth
333
+ 2023-05-03 14:18:02,296 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4300.pth
334
+ 2023-05-03 14:18:29,408 44k INFO ====> Epoch: 64, cost 54.57 s
335
+ 2023-05-03 14:19:07,204 44k INFO Train Epoch: 65 [77%]
336
+ 2023-05-03 14:19:07,205 44k INFO Losses: [2.2484593391418457, 2.5644028186798096, 11.485355377197266, 21.17015838623047, 1.1813242435455322], step: 4600, lr: 9.92031418779886e-05, reference_loss: 38.64970016479492
337
+ 2023-05-03 14:19:12,347 44k INFO Saving model and optimizer state at iteration 65 to ./logs/44k/G_4600.pth
338
+ 2023-05-03 14:19:14,047 44k INFO Saving model and optimizer state at iteration 65 to ./logs/44k/D_4600.pth
339
+ 2023-05-03 14:19:14,583 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4400.pth
340
+ 2023-05-03 14:19:14,611 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4400.pth
341
+ 2023-05-03 14:19:24,026 44k INFO ====> Epoch: 65, cost 54.62 s
342
+ 2023-05-03 14:20:11,109 44k INFO ====> Epoch: 66, cost 47.08 s
343
+ 2023-05-03 14:20:23,165 44k INFO Train Epoch: 67 [18%]
344
+ 2023-05-03 14:20:23,166 44k INFO Losses: [2.3424651622772217, 2.569873332977295, 7.83887243270874, 22.892330169677734, 1.65440034866333], step: 4700, lr: 9.917834264256819e-05, reference_loss: 37.29793930053711
345
+ 2023-05-03 14:20:28,353 44k INFO Saving model and optimizer state at iteration 67 to ./logs/44k/G_4700.pth
346
+ 2023-05-03 14:20:30,072 44k INFO Saving model and optimizer state at iteration 67 to ./logs/44k/D_4700.pth
347
+ 2023-05-03 14:20:30,606 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4500.pth
348
+ 2023-05-03 14:20:30,635 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4500.pth
349
+ 2023-05-03 14:21:05,218 44k INFO ====> Epoch: 67, cost 54.11 s
350
+ 2023-05-03 14:21:35,022 44k INFO Train Epoch: 68 [59%]
351
+ 2023-05-03 14:21:35,023 44k INFO Losses: [2.3991572856903076, 2.080165386199951, 10.898751258850098, 20.9050235748291, 0.7145375609397888], step: 4800, lr: 9.916594534973787e-05, reference_loss: 36.99763488769531
352
+ 2023-05-03 14:21:40,023 44k INFO Saving model and optimizer state at iteration 68 to ./logs/44k/G_4800.pth
353
+ 2023-05-03 14:21:41,821 44k INFO Saving model and optimizer state at iteration 68 to ./logs/44k/D_4800.pth
354
+ 2023-05-03 14:21:42,354 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4600.pth
355
+ 2023-05-03 14:21:42,383 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4600.pth
356
+ 2023-05-03 14:21:59,598 44k INFO ====> Epoch: 68, cost 54.38 s
357
+ 2023-05-03 14:22:46,885 44k INFO ====> Epoch: 69, cost 47.29 s
358
+ 2023-05-03 14:22:51,107 44k INFO Train Epoch: 70 [0%]
359
+ 2023-05-03 14:22:51,108 44k INFO Losses: [2.243220329284668, 2.3275976181030273, 9.493850708007812, 20.528539657592773, 1.0689058303833008], step: 4900, lr: 9.914115541286833e-05, reference_loss: 35.662113189697266
360
+ 2023-05-03 14:22:56,175 44k INFO Saving model and optimizer state at iteration 70 to ./logs/44k/G_4900.pth
361
+ 2023-05-03 14:22:57,868 44k INFO Saving model and optimizer state at iteration 70 to ./logs/44k/D_4900.pth
362
+ 2023-05-03 14:22:58,400 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4700.pth
363
+ 2023-05-03 14:22:58,429 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4700.pth
364
+ 2023-05-03 14:23:41,035 44k INFO ====> Epoch: 70, cost 54.15 s
365
+ 2023-05-03 14:24:04,991 44k INFO Train Epoch: 71 [41%]
366
+ 2023-05-03 14:24:04,992 44k INFO Losses: [2.3157591819763184, 2.332456588745117, 7.5815205574035645, 22.40475845336914, 1.2106596231460571], step: 5000, lr: 9.912876276844171e-05, reference_loss: 35.84515380859375
367
+ 2023-05-03 14:24:10,111 44k INFO Saving model and optimizer state at iteration 71 to ./logs/44k/G_5000.pth
368
+ 2023-05-03 14:24:11,823 44k INFO Saving model and optimizer state at iteration 71 to ./logs/44k/D_5000.pth
369
+ 2023-05-03 14:24:12,362 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4800.pth
370
+ 2023-05-03 14:24:12,391 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4800.pth
371
+ 2023-05-03 14:24:40,346 44k INFO ====> Epoch: 71, cost 59.31 s
372
+ 2023-05-03 14:25:19,763 44k INFO Train Epoch: 72 [82%]
373
+ 2023-05-03 14:25:19,763 44k INFO Losses: [2.090325117111206, 2.388118267059326, 12.023096084594727, 22.705028533935547, 1.1269657611846924], step: 5100, lr: 9.911637167309565e-05, reference_loss: 40.33353042602539
374
+ 2023-05-03 14:25:25,025 44k INFO Saving model and optimizer state at iteration 72 to ./logs/44k/G_5100.pth
375
+ 2023-05-03 14:25:26,635 44k INFO Saving model and optimizer state at iteration 72 to ./logs/44k/D_5100.pth
376
+ 2023-05-03 14:25:27,169 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4900.pth
377
+ 2023-05-03 14:25:27,198 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4900.pth
378
+ 2023-05-03 14:25:34,804 44k INFO ====> Epoch: 72, cost 54.46 s
379
+ 2023-05-03 14:26:21,926 44k INFO ====> Epoch: 73, cost 47.12 s
380
+ 2023-05-03 14:26:35,964 44k INFO Train Epoch: 74 [23%]
381
+ 2023-05-03 14:26:35,965 44k INFO Losses: [2.2803421020507812, 2.3115463256835938, 10.413973808288574, 23.707111358642578, 1.475788950920105], step: 5200, lr: 9.909159412887068e-05, reference_loss: 40.18876266479492
382
+ 2023-05-03 14:26:41,101 44k INFO Saving model and optimizer state at iteration 74 to ./logs/44k/G_5200.pth
383
+ 2023-05-03 14:26:42,833 44k INFO Saving model and optimizer state at iteration 74 to ./logs/44k/D_5200.pth
384
+ 2023-05-03 14:26:43,370 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5000.pth
385
+ 2023-05-03 14:26:43,398 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5000.pth
386
+ 2023-05-03 14:27:16,107 44k INFO ====> Epoch: 74, cost 54.18 s
387
+ 2023-05-03 14:27:47,480 44k INFO Train Epoch: 75 [63%]
388
+ 2023-05-03 14:27:47,481 44k INFO Losses: [2.234922409057617, 2.467733860015869, 9.63402271270752, 20.404632568359375, 1.2089588642120361], step: 5300, lr: 9.907920767960457e-05, reference_loss: 35.95026779174805
389
+ 2023-05-03 14:27:52,458 44k INFO Saving model and optimizer state at iteration 75 to ./logs/44k/G_5300.pth
390
+ 2023-05-03 14:27:54,283 44k INFO Saving model and optimizer state at iteration 75 to ./logs/44k/D_5300.pth
391
+ 2023-05-03 14:27:54,817 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5100.pth
392
+ 2023-05-03 14:27:54,845 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5100.pth
393
+ 2023-05-03 14:28:10,119 44k INFO ====> Epoch: 75, cost 54.01 s
394
+ 2023-05-03 14:28:56,847 44k INFO ====> Epoch: 76, cost 46.73 s
395
+ 2023-05-03 14:29:02,893 44k INFO Train Epoch: 77 [4%]
396
+ 2023-05-03 14:29:02,893 44k INFO Losses: [2.6155261993408203, 2.135758876800537, 5.038893222808838, 17.311443328857422, 1.409765601158142], step: 5400, lr: 9.905443942579728e-05, reference_loss: 28.51138687133789
397
+ 2023-05-03 14:29:07,925 44k INFO Saving model and optimizer state at iteration 77 to ./logs/44k/G_5400.pth
398
+ 2023-05-03 14:29:09,665 44k INFO Saving model and optimizer state at iteration 77 to ./logs/44k/D_5400.pth
399
+ 2023-05-03 14:29:10,192 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5200.pth
400
+ 2023-05-03 14:29:10,221 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5200.pth
401
+ 2023-05-03 14:29:51,047 44k INFO ====> Epoch: 77, cost 54.20 s
402
+ 2023-05-03 14:30:14,772 44k INFO Train Epoch: 78 [45%]
403
+ 2023-05-03 14:30:14,772 44k INFO Losses: [2.413071393966675, 2.4141740798950195, 6.657437801361084, 20.914470672607422, 1.1908607482910156], step: 5500, lr: 9.904205762086905e-05, reference_loss: 33.59001541137695
404
+ 2023-05-03 14:30:20,010 44k INFO Saving model and optimizer state at iteration 78 to ./logs/44k/G_5500.pth
405
+ 2023-05-03 14:30:21,621 44k INFO Saving model and optimizer state at iteration 78 to ./logs/44k/D_5500.pth
406
+ 2023-05-03 14:30:22,193 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5300.pth
407
+ 2023-05-03 14:30:22,222 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5300.pth
408
+ 2023-05-03 14:30:45,550 44k INFO ====> Epoch: 78, cost 54.50 s
409
+ 2023-05-03 14:31:29,477 44k INFO Train Epoch: 79 [86%]
410
+ 2023-05-03 14:31:29,478 44k INFO Losses: [2.4432146549224854, 2.3251218795776367, 12.018153190612793, 20.31390380859375, 1.342065453529358], step: 5600, lr: 9.902967736366644e-05, reference_loss: 38.44245910644531
411
+ 2023-05-03 14:31:34,567 44k INFO Saving model and optimizer state at iteration 79 to ./logs/44k/G_5600.pth
412
+ 2023-05-03 14:31:36,333 44k INFO Saving model and optimizer state at iteration 79 to ./logs/44k/D_5600.pth
413
+ 2023-05-03 14:31:36,901 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5400.pth
414
+ 2023-05-03 14:31:36,929 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5400.pth
415
+ 2023-05-03 14:31:42,708 44k INFO ====> Epoch: 79, cost 57.16 s
416
+ 2023-05-03 14:32:34,654 44k INFO ====> Epoch: 80, cost 51.95 s
417
+ 2023-05-03 14:32:50,414 44k INFO Train Epoch: 81 [27%]
418
+ 2023-05-03 14:32:50,415 44k INFO Losses: [2.7789437770843506, 2.3213107585906982, 9.386683464050293, 21.96425437927246, 1.3818237781524658], step: 5700, lr: 9.900492149166423e-05, reference_loss: 37.8330192565918
419
+ 2023-05-03 14:32:55,474 44k INFO Saving model and optimizer state at iteration 81 to ./logs/44k/G_5700.pth
420
+ 2023-05-03 14:32:57,143 44k INFO Saving model and optimizer state at iteration 81 to ./logs/44k/D_5700.pth
421
+ 2023-05-03 14:32:57,673 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5500.pth
422
+ 2023-05-03 14:32:57,702 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5500.pth
423
+ 2023-05-03 14:33:28,696 44k INFO ====> Epoch: 81, cost 54.04 s
424
+ 2023-05-03 14:34:01,815 44k INFO Train Epoch: 82 [68%]
425
+ 2023-05-03 14:34:01,816 44k INFO Losses: [2.211815357208252, 2.320291042327881, 5.064887046813965, 23.727109909057617, 0.9825482964515686], step: 5800, lr: 9.899254587647776e-05, reference_loss: 34.3066520690918
426
+ 2023-05-03 14:34:06,827 44k INFO Saving model and optimizer state at iteration 82 to ./logs/44k/G_5800.pth
427
+ 2023-05-03 14:34:08,521 44k INFO Saving model and optimizer state at iteration 82 to ./logs/44k/D_5800.pth
428
+ 2023-05-03 14:34:09,055 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5600.pth
429
+ 2023-05-03 14:34:09,104 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5600.pth
430
+ 2023-05-03 14:34:22,816 44k INFO ====> Epoch: 82, cost 54.12 s
431
+ 2023-05-03 14:35:10,041 44k INFO ====> Epoch: 83, cost 47.23 s
432
+ 2023-05-03 14:35:17,956 44k INFO Train Epoch: 84 [8%]
433
+ 2023-05-03 14:35:17,957 44k INFO Losses: [2.484198570251465, 2.2484796047210693, 10.755833625793457, 20.941932678222656, 1.2315059900283813], step: 5900, lr: 9.896779928676716e-05, reference_loss: 37.661949157714844
434
+ 2023-05-03 14:35:23,191 44k INFO Saving model and optimizer state at iteration 84 to ./logs/44k/G_5900.pth
435
+ 2023-05-03 14:35:24,926 44k INFO Saving model and optimizer state at iteration 84 to ./logs/44k/D_5900.pth
436
+ 2023-05-03 14:35:25,506 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5700.pth
437
+ 2023-05-03 14:35:25,534 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5700.pth
438
+ 2023-05-03 14:36:04,956 44k INFO ====> Epoch: 84, cost 54.91 s
439
+ 2023-05-03 14:36:30,506 44k INFO Train Epoch: 85 [49%]
440
+ 2023-05-03 14:36:30,507 44k INFO Losses: [2.388766288757324, 2.1078274250030518, 8.077157020568848, 20.986013412475586, 1.242616891860962], step: 6000, lr: 9.895542831185631e-05, reference_loss: 34.80238342285156
441
+ 2023-05-03 14:36:35,688 44k INFO Saving model and optimizer state at iteration 85 to ./logs/44k/G_6000.pth
442
+ 2023-05-03 14:36:37,309 44k INFO Saving model and optimizer state at iteration 85 to ./logs/44k/D_6000.pth
443
+ 2023-05-03 14:36:37,844 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5800.pth
444
+ 2023-05-03 14:36:37,875 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5800.pth
445
+ 2023-05-03 14:36:59,366 44k INFO ====> Epoch: 85, cost 54.41 s
446
+ 2023-05-03 14:37:42,269 44k INFO Train Epoch: 86 [90%]
447
+ 2023-05-03 14:37:42,270 44k INFO Losses: [2.4167308807373047, 2.440798044204712, 8.85080623626709, 20.857934951782227, 1.072945237159729], step: 6100, lr: 9.894305888331732e-05, reference_loss: 35.63921356201172
448
+ 2023-05-03 14:37:47,385 44k INFO Saving model and optimizer state at iteration 86 to ./logs/44k/G_6100.pth
449
+ 2023-05-03 14:37:49,228 44k INFO Saving model and optimizer state at iteration 86 to ./logs/44k/D_6100.pth
450
+ 2023-05-03 14:37:49,764 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5900.pth
451
+ 2023-05-03 14:37:49,810 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5900.pth
452
+ 2023-05-03 14:37:53,883 44k INFO ====> Epoch: 86, cost 54.52 s
453
+ 2023-05-03 14:38:40,995 44k INFO ====> Epoch: 87, cost 47.11 s
454
+ 2023-05-03 14:38:58,697 44k INFO Train Epoch: 88 [31%]
455
+ 2023-05-03 14:38:58,698 44k INFO Losses: [2.289813995361328, 2.1840624809265137, 11.359431266784668, 23.120685577392578, 1.0711365938186646], step: 6200, lr: 9.891832466458178e-05, reference_loss: 40.02513122558594
456
+ 2023-05-03 14:39:03,798 44k INFO Saving model and optimizer state at iteration 88 to ./logs/44k/G_6200.pth
457
+ 2023-05-03 14:39:05,398 44k INFO Saving model and optimizer state at iteration 88 to ./logs/44k/D_6200.pth
458
+ 2023-05-03 14:39:05,961 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6000.pth
459
+ 2023-05-03 14:39:05,992 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6000.pth
460
+ 2023-05-03 14:39:35,106 44k INFO ====> Epoch: 88, cost 54.11 s
461
+ 2023-05-03 14:40:11,101 44k INFO Train Epoch: 89 [72%]
462
+ 2023-05-03 14:40:11,102 44k INFO Losses: [2.3503124713897705, 2.366110324859619, 7.645803451538086, 23.64559555053711, 1.1122896671295166], step: 6300, lr: 9.89059598739987e-05, reference_loss: 37.12010955810547
463
+ 2023-05-03 14:40:16,443 44k INFO Saving model and optimizer state at iteration 89 to ./logs/44k/G_6300.pth
464
+ 2023-05-03 14:40:18,088 44k INFO Saving model and optimizer state at iteration 89 to ./logs/44k/D_6300.pth
465
+ 2023-05-03 14:40:18,627 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6100.pth
466
+ 2023-05-03 14:40:18,657 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6100.pth
467
+ 2023-05-03 14:40:30,437 44k INFO ====> Epoch: 89, cost 55.33 s
468
+ 2023-05-03 14:41:17,208 44k INFO ====> Epoch: 90, cost 46.77 s
469
+ 2023-05-03 14:41:26,818 44k INFO Train Epoch: 91 [13%]
470
+ 2023-05-03 14:41:26,818 44k INFO Losses: [2.2005672454833984, 2.5051355361938477, 9.279576301574707, 22.79197120666504, 1.0933576822280884], step: 6400, lr: 9.888123492943583e-05, reference_loss: 37.870609283447266
471
+ 2023-05-03 14:41:32,032 44k INFO Saving model and optimizer state at iteration 91 to ./logs/44k/G_6400.pth
472
+ 2023-05-03 14:41:33,728 44k INFO Saving model and optimizer state at iteration 91 to ./logs/44k/D_6400.pth
473
+ 2023-05-03 14:41:34,259 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6200.pth
474
+ 2023-05-03 14:41:34,307 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6200.pth
475
+ 2023-05-03 14:42:11,194 44k INFO ====> Epoch: 91, cost 53.99 s
476
+ 2023-05-03 14:42:38,796 44k INFO Train Epoch: 92 [54%]
477
+ 2023-05-03 14:42:38,796 44k INFO Losses: [2.4646763801574707, 2.076392412185669, 7.404085159301758, 21.058698654174805, 1.0544928312301636], step: 6500, lr: 9.886887477506964e-05, reference_loss: 34.058345794677734
478
+ 2023-05-03 14:42:43,951 44k INFO Saving model and optimizer state at iteration 92 to ./logs/44k/G_6500.pth
479
+ 2023-05-03 14:42:45,633 44k INFO Saving model and optimizer state at iteration 92 to ./logs/44k/D_6500.pth
480
+ 2023-05-03 14:42:46,317 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6300.pth
481
+ 2023-05-03 14:42:46,348 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6300.pth
482
+ 2023-05-03 14:43:06,039 44k INFO ====> Epoch: 92, cost 54.85 s
483
+ 2023-05-03 14:43:55,537 44k INFO Train Epoch: 93 [94%]
484
+ 2023-05-03 14:43:55,537 44k INFO Losses: [2.5010299682617188, 2.3011274337768555, 8.138545036315918, 23.67609405517578, 0.9986937642097473], step: 6600, lr: 9.885651616572276e-05, reference_loss: 37.61549377441406
485
+ 2023-05-03 14:44:00,682 44k INFO Saving model and optimizer state at iteration 93 to ./logs/44k/G_6600.pth
486
+ 2023-05-03 14:44:02,367 44k INFO Saving model and optimizer state at iteration 93 to ./logs/44k/D_6600.pth
487
+ 2023-05-03 14:44:02,916 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6400.pth
488
+ 2023-05-03 14:44:02,945 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6400.pth
489
+ 2023-05-03 14:44:05,398 44k INFO ====> Epoch: 93, cost 59.36 s
490
+ 2023-05-03 14:44:52,822 44k INFO ====> Epoch: 94, cost 47.42 s
491
+ 2023-05-03 14:45:12,476 44k INFO Train Epoch: 95 [35%]
492
+ 2023-05-03 14:45:12,477 44k INFO Losses: [2.310382127761841, 2.2262511253356934, 8.078502655029297, 19.226985931396484, 1.524962306022644], step: 6700, lr: 9.883180358131438e-05, reference_loss: 33.36708450317383
493
+ 2023-05-03 14:45:17,573 44k INFO Saving model and optimizer state at iteration 95 to ./logs/44k/G_6700.pth
494
+ 2023-05-03 14:45:19,270 44k INFO Saving model and optimizer state at iteration 95 to ./logs/44k/D_6700.pth
495
+ 2023-05-03 14:45:19,801 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6500.pth
496
+ 2023-05-03 14:45:19,850 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6500.pth
497
+ 2023-05-03 14:45:47,682 44k INFO ====> Epoch: 95, cost 54.86 s
498
+ 2023-05-03 14:46:25,727 44k INFO Train Epoch: 96 [76%]
499
+ 2023-05-03 14:46:25,728 44k INFO Losses: [2.6544623374938965, 2.2764406204223633, 6.531411170959473, 19.763704299926758, 1.2900036573410034], step: 6800, lr: 9.881944960586671e-05, reference_loss: 32.516021728515625
500
+ 2023-05-03 14:46:31,277 44k INFO Saving model and optimizer state at iteration 96 to ./logs/44k/G_6800.pth
501
+ 2023-05-03 14:46:33,069 44k INFO Saving model and optimizer state at iteration 96 to ./logs/44k/D_6800.pth
502
+ 2023-05-03 14:46:33,648 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6600.pth
503
+ 2023-05-03 14:46:33,677 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6600.pth
504
+ 2023-05-03 14:46:43,773 44k INFO ====> Epoch: 96, cost 56.09 s
505
+ 2023-05-03 14:47:30,320 44k INFO ====> Epoch: 97, cost 46.55 s
506
+ 2023-05-03 14:47:41,778 44k INFO Train Epoch: 98 [17%]
507
+ 2023-05-03 14:47:41,779 44k INFO Losses: [2.525698184967041, 2.3287713527679443, 8.644176483154297, 19.613706588745117, 1.2030137777328491], step: 6900, lr: 9.879474628751914e-05, reference_loss: 34.315364837646484
508
+ 2023-05-03 14:47:47,031 44k INFO Saving model and optimizer state at iteration 98 to ./logs/44k/G_6900.pth
509
+ 2023-05-03 14:47:48,723 44k INFO Saving model and optimizer state at iteration 98 to ./logs/44k/D_6900.pth
510
+ 2023-05-03 14:47:49,257 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6700.pth
511
+ 2023-05-03 14:47:49,286 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6700.pth
512
+ 2023-05-03 14:48:24,690 44k INFO ====> Epoch: 98, cost 54.37 s
513
+ 2023-05-03 14:48:53,995 44k INFO Train Epoch: 99 [58%]
514
+ 2023-05-03 14:48:53,996 44k INFO Losses: [2.533390522003174, 2.219142436981201, 7.681521892547607, 18.070770263671875, 1.1386370658874512], step: 7000, lr: 9.87823969442332e-05, reference_loss: 31.643463134765625
515
+ 2023-05-03 14:48:59,102 44k INFO Saving model and optimizer state at iteration 99 to ./logs/44k/G_7000.pth
516
+ 2023-05-03 14:49:00,957 44k INFO Saving model and optimizer state at iteration 99 to ./logs/44k/D_7000.pth
517
+ 2023-05-03 14:49:01,520 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6800.pth
518
+ 2023-05-03 14:49:01,574 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6800.pth
519
+ 2023-05-03 14:49:19,402 44k INFO ====> Epoch: 99, cost 54.71 s
520
+ 2023-05-03 14:50:12,441 44k INFO Train Epoch: 100 [99%]
521
+ 2023-05-03 14:50:12,442 44k INFO Losses: [2.4274239540100098, 2.672248601913452, 8.535931587219238, 19.511117935180664, 1.0394376516342163], step: 7100, lr: 9.877004914461517e-05, reference_loss: 34.1861572265625
522
+ 2023-05-03 14:50:17,757 44k INFO Saving model and optimizer state at iteration 100 to ./logs/44k/G_7100.pth
523
+ 2023-05-03 14:50:19,557 44k INFO Saving model and optimizer state at iteration 100 to ./logs/44k/D_7100.pth
524
+ 2023-05-03 14:50:20,132 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6900.pth
525
+ 2023-05-03 14:50:20,163 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6900.pth
526
+ 2023-05-03 14:50:20,732 44k INFO ====> Epoch: 100, cost 61.33 s
527
+ 2023-05-03 14:51:12,740 44k INFO ====> Epoch: 101, cost 52.01 s
528
+ 2023-05-03 14:51:33,881 44k INFO Train Epoch: 102 [39%]
529
+ 2023-05-03 14:51:33,882 44k INFO Losses: [2.2989039421081543, 2.5206449031829834, 8.606928825378418, 21.207876205444336, 1.165399193763733], step: 7200, lr: 9.874535817561101e-05, reference_loss: 35.79975128173828
530
+ 2023-05-03 14:51:39,061 44k INFO Saving model and optimizer state at iteration 102 to ./logs/44k/G_7200.pth
531
+ 2023-05-03 14:51:40,726 44k INFO Saving model and optimizer state at iteration 102 to ./logs/44k/D_7200.pth
532
+ 2023-05-03 14:51:41,279 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7000.pth
533
+ 2023-05-03 14:51:41,309 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7000.pth
534
+ 2023-05-03 14:52:06,869 44k INFO ====> Epoch: 102, cost 54.13 s
535
+ 2023-05-03 14:52:45,514 44k INFO Train Epoch: 103 [80%]
536
+ 2023-05-03 14:52:45,515 44k INFO Losses: [2.3017122745513916, 2.422215223312378, 9.920147895812988, 18.564119338989258, 0.645395040512085], step: 7300, lr: 9.873301500583906e-05, reference_loss: 33.85358810424805
537
+ 2023-05-03 14:52:50,524 44k INFO Saving model and optimizer state at iteration 103 to ./logs/44k/G_7300.pth
538
+ 2023-05-03 14:52:52,344 44k INFO Saving model and optimizer state at iteration 103 to ./logs/44k/D_7300.pth
539
+ 2023-05-03 14:52:52,900 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7100.pth
540
+ 2023-05-03 14:52:52,953 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7100.pth
541
+ 2023-05-03 14:53:01,109 44k INFO ====> Epoch: 103, cost 54.24 s
542
+ 2023-05-03 14:53:47,732 44k INFO ====> Epoch: 104, cost 46.62 s
543
+ 2023-05-03 14:54:01,060 44k INFO Train Epoch: 105 [21%]
544
+ 2023-05-03 14:54:01,060 44k INFO Losses: [2.75062894821167, 2.412715196609497, 10.205183029174805, 20.36348533630371, 1.4236520528793335], step: 7400, lr: 9.870833329479095e-05, reference_loss: 37.15566635131836
545
+ 2023-05-03 14:54:06,286 44k INFO Saving model and optimizer state at iteration 105 to ./logs/44k/G_7400.pth
546
+ 2023-05-03 14:54:07,938 44k INFO Saving model and optimizer state at iteration 105 to ./logs/44k/D_7400.pth
547
+ 2023-05-03 14:54:08,493 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7200.pth
548
+ 2023-05-03 14:54:08,534 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7200.pth
549
+ 2023-05-03 14:54:41,911 44k INFO ====> Epoch: 105, cost 54.18 s
550
+ 2023-05-03 14:55:13,018 44k INFO Train Epoch: 106 [62%]
551
+ 2023-05-03 14:55:13,019 44k INFO Losses: [2.3272743225097656, 2.4755334854125977, 9.030518531799316, 21.309324264526367, 0.9096031188964844], step: 7500, lr: 9.86959947531291e-05, reference_loss: 36.05225372314453
552
+ 2023-05-03 14:55:18,285 44k INFO Saving model and optimizer state at iteration 106 to ./logs/44k/G_7500.pth
553
+ 2023-05-03 14:55:20,026 44k INFO Saving model and optimizer state at iteration 106 to ./logs/44k/D_7500.pth
554
+ 2023-05-03 14:55:20,610 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7300.pth
555
+ 2023-05-03 14:55:20,639 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7300.pth
556
+ 2023-05-03 14:55:38,546 44k INFO ====> Epoch: 106, cost 56.63 s
557
+ 2023-05-03 14:56:25,232 44k INFO ====> Epoch: 107, cost 46.69 s
558
+ 2023-05-03 14:56:30,729 44k INFO Train Epoch: 108 [3%]
559
+ 2023-05-03 14:56:30,730 44k INFO Losses: [2.4331343173980713, 2.1562960147857666, 10.360976219177246, 20.71307945251465, 1.2649202346801758], step: 7600, lr: 9.867132229656573e-05, reference_loss: 36.92840576171875
560
+ 2023-05-03 14:56:35,983 44k INFO Saving model and optimizer state at iteration 108 to ./logs/44k/G_7600.pth
561
+ 2023-05-03 14:56:37,594 44k INFO Saving model and optimizer state at iteration 108 to ./logs/44k/D_7600.pth
562
+ 2023-05-03 14:56:38,147 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7400.pth
563
+ 2023-05-03 14:56:38,199 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7400.pth
564
+ 2023-05-03 14:57:19,302 44k INFO ====> Epoch: 108, cost 54.07 s
565
+ 2023-05-03 14:57:42,227 44k INFO Train Epoch: 109 [44%]
566
+ 2023-05-03 14:57:42,228 44k INFO Losses: [2.3168601989746094, 2.5806796550750732, 9.585286140441895, 20.979799270629883, 1.159220814704895], step: 7700, lr: 9.865898838127865e-05, reference_loss: 36.62184524536133
567
+ 2023-05-03 14:57:47,470 44k INFO Saving model and optimizer state at iteration 109 to ./logs/44k/G_7700.pth
568
+ 2023-05-03 14:57:49,300 44k INFO Saving model and optimizer state at iteration 109 to ./logs/44k/D_7700.pth
569
+ 2023-05-03 14:57:49,857 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7500.pth
570
+ 2023-05-03 14:57:49,899 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7500.pth
571
+ 2023-05-03 14:58:13,625 44k INFO ====> Epoch: 109, cost 54.32 s
572
+ 2023-05-03 14:58:54,066 44k INFO Train Epoch: 110 [85%]
573
+ 2023-05-03 14:58:54,067 44k INFO Losses: [2.639904260635376, 2.3363561630249023, 9.849010467529297, 19.11192512512207, 1.1576781272888184], step: 7800, lr: 9.864665600773098e-05, reference_loss: 35.094871520996094
574
+ 2023-05-03 14:58:59,104 44k INFO Saving model and optimizer state at iteration 110 to ./logs/44k/G_7800.pth
575
+ 2023-05-03 14:59:00,792 44k INFO Saving model and optimizer state at iteration 110 to ./logs/44k/D_7800.pth
576
+ 2023-05-03 14:59:01,344 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7600.pth
577
+ 2023-05-03 14:59:01,372 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7600.pth
578
+ 2023-05-03 14:59:07,747 44k INFO ====> Epoch: 110, cost 54.12 s
579
+ 2023-05-03 14:59:59,651 44k INFO ====> Epoch: 111, cost 51.90 s
580
+ 2023-05-03 15:00:14,927 44k INFO Train Epoch: 112 [25%]
581
+ 2023-05-03 15:00:14,928 44k INFO Losses: [2.4285850524902344, 2.606787919998169, 11.0058012008667, 19.486034393310547, 0.8991149663925171], step: 7900, lr: 9.862199588508305e-05, reference_loss: 36.42632293701172
582
+ 2023-05-03 15:00:20,191 44k INFO Saving model and optimizer state at iteration 112 to ./logs/44k/G_7900.pth
583
+ 2023-05-03 15:00:21,924 44k INFO Saving model and optimizer state at iteration 112 to ./logs/44k/D_7900.pth
584
+ 2023-05-03 15:00:22,486 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7700.pth
585
+ 2023-05-03 15:00:22,538 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7700.pth
586
+ 2023-05-03 15:00:54,314 44k INFO ====> Epoch: 112, cost 54.66 s
587
+ 2023-05-03 15:01:27,150 44k INFO Train Epoch: 113 [66%]
588
+ 2023-05-03 15:01:27,151 44k INFO Losses: [2.523535966873169, 2.58498477935791, 7.795136451721191, 22.24933624267578, 1.2834935188293457], step: 8000, lr: 9.86096681355974e-05, reference_loss: 36.436485290527344
589
+ 2023-05-03 15:01:32,472 44k INFO Saving model and optimizer state at iteration 113 to ./logs/44k/G_8000.pth
590
+ 2023-05-03 15:01:34,307 44k INFO Saving model and optimizer state at iteration 113 to ./logs/44k/D_8000.pth
591
+ 2023-05-03 15:01:34,875 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7800.pth
592
+ 2023-05-03 15:01:34,917 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7800.pth
593
+ 2023-05-03 15:01:49,136 44k INFO ====> Epoch: 113, cost 54.82 s
594
+ 2023-05-03 15:02:36,217 44k INFO ====> Epoch: 114, cost 47.08 s
595
+ 2023-05-03 15:02:43,597 44k INFO Train Epoch: 115 [7%]
596
+ 2023-05-03 15:02:43,598 44k INFO Losses: [2.389103889465332, 2.183370351791382, 8.995611190795898, 17.36598014831543, 0.7324305176734924], step: 8100, lr: 9.858501725933955e-05, reference_loss: 31.6664981842041
597
+ 2023-05-03 15:02:48,652 44k INFO Saving model and optimizer state at iteration 115 to ./logs/44k/G_8100.pth
598
+ 2023-05-03 15:02:50,350 44k INFO Saving model and optimizer state at iteration 115 to ./logs/44k/D_8100.pth
599
+ 2023-05-03 15:02:50,908 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7900.pth
600
+ 2023-05-03 15:02:50,940 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7900.pth
601
+ 2023-05-03 15:03:30,855 44k INFO ====> Epoch: 115, cost 54.64 s
602
+ 2023-05-03 15:03:55,782 44k INFO Train Epoch: 116 [48%]
603
+ 2023-05-03 15:03:55,783 44k INFO Losses: [2.1408324241638184, 2.1866962909698486, 9.188668251037598, 22.545970916748047, 1.038486123085022], step: 8200, lr: 9.857269413218213e-05, reference_loss: 37.10065460205078
604
+ 2023-05-03 15:04:00,808 44k INFO Saving model and optimizer state at iteration 116 to ./logs/44k/G_8200.pth
605
+ 2023-05-03 15:04:02,446 44k INFO Saving model and optimizer state at iteration 116 to ./logs/44k/D_8200.pth
606
+ 2023-05-03 15:04:03,008 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8000.pth
607
+ 2023-05-03 15:04:03,060 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8000.pth
608
+ 2023-05-03 15:04:25,220 44k INFO ====> Epoch: 116, cost 54.36 s
609
+ 2023-05-03 15:05:07,296 44k INFO Train Epoch: 117 [89%]
610
+ 2023-05-03 15:05:07,297 44k INFO Losses: [2.123523712158203, 2.3391237258911133, 10.087421417236328, 21.18076515197754, 1.0259798765182495], step: 8300, lr: 9.85603725454156e-05, reference_loss: 36.756813049316406
611
+ 2023-05-03 15:05:12,704 44k INFO Saving model and optimizer state at iteration 117 to ./logs/44k/G_8300.pth
612
+ 2023-05-03 15:05:14,431 44k INFO Saving model and optimizer state at iteration 117 to ./logs/44k/D_8300.pth
613
+ 2023-05-03 15:05:14,990 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8100.pth
614
+ 2023-05-03 15:05:15,033 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8100.pth
615
+ 2023-05-03 15:05:19,669 44k INFO ====> Epoch: 117, cost 54.45 s
616
+ 2023-05-03 15:06:06,258 44k INFO ====> Epoch: 118, cost 46.59 s
617
+ 2023-05-03 15:06:23,373 44k INFO Train Epoch: 119 [30%]
618
+ 2023-05-03 15:06:23,374 44k INFO Losses: [2.2387447357177734, 2.410027027130127, 12.253347396850586, 22.95694923400879, 0.9085140228271484], step: 8400, lr: 9.853573399228505e-05, reference_loss: 40.76758575439453
619
+ 2023-05-03 15:06:28,624 44k INFO Saving model and optimizer state at iteration 119 to ./logs/44k/G_8400.pth
620
+ 2023-05-03 15:06:30,293 44k INFO Saving model and optimizer state at iteration 119 to ./logs/44k/D_8400.pth
621
+ 2023-05-03 15:06:30,852 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8200.pth
622
+ 2023-05-03 15:06:30,883 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8200.pth
623
+ 2023-05-03 15:07:00,915 44k INFO ====> Epoch: 119, cost 54.66 s
624
+ 2023-05-03 15:07:35,666 44k INFO Train Epoch: 120 [70%]
625
+ 2023-05-03 15:07:35,667 44k INFO Losses: [2.337846279144287, 2.437479019165039, 10.248119354248047, 18.765079498291016, 0.8664497137069702], step: 8500, lr: 9.8523417025536e-05, reference_loss: 34.65497589111328
626
+ 2023-05-03 15:07:40,791 44k INFO Saving model and optimizer state at iteration 120 to ./logs/44k/G_8500.pth
627
+ 2023-05-03 15:07:42,553 44k INFO Saving model and optimizer state at iteration 120 to ./logs/44k/D_8500.pth
628
+ 2023-05-03 15:07:43,115 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8300.pth
629
+ 2023-05-03 15:07:43,168 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8300.pth
630
+ 2023-05-03 15:07:55,625 44k INFO ====> Epoch: 120, cost 54.71 s
631
+ 2023-05-03 15:08:47,327 44k INFO ====> Epoch: 121, cost 51.70 s
632
+ 2023-05-03 15:08:56,353 44k INFO Train Epoch: 122 [11%]
633
+ 2023-05-03 15:08:56,354 44k INFO Losses: [2.2986395359039307, 2.452651262283325, 10.624441146850586, 18.602886199951172, 1.0152174234390259], step: 8600, lr: 9.8498787710708e-05, reference_loss: 34.99383544921875
634
+ 2023-05-03 15:09:01,490 44k INFO Saving model and optimizer state at iteration 122 to ./logs/44k/G_8600.pth
635
+ 2023-05-03 15:09:03,203 44k INFO Saving model and optimizer state at iteration 122 to ./logs/44k/D_8600.pth
636
+ 2023-05-03 15:09:03,760 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8400.pth
637
+ 2023-05-03 15:09:03,802 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8400.pth
638
+ 2023-05-03 15:09:41,161 44k INFO ====> Epoch: 122, cost 53.83 s
639
+ 2023-05-03 15:10:07,424 44k INFO Train Epoch: 123 [52%]
640
+ 2023-05-03 15:10:07,424 44k INFO Losses: [2.413036823272705, 2.3398640155792236, 10.38782024383545, 20.648679733276367, 0.8482047319412231], step: 8700, lr: 9.848647536224416e-05, reference_loss: 36.63760757446289
641
+ 2023-05-03 15:10:12,546 44k INFO Saving model and optimizer state at iteration 123 to ./logs/44k/G_8700.pth
642
+ 2023-05-03 15:10:14,214 44k INFO Saving model and optimizer state at iteration 123 to ./logs/44k/D_8700.pth
643
+ 2023-05-03 15:10:14,772 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8500.pth
644
+ 2023-05-03 15:10:14,803 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8500.pth
645
+ 2023-05-03 15:10:34,644 44k INFO ====> Epoch: 123, cost 53.48 s
646
+ 2023-05-03 15:11:18,669 44k INFO Train Epoch: 124 [93%]
647
+ 2023-05-03 15:11:18,669 44k INFO Losses: [2.443486213684082, 2.4561853408813477, 8.95302677154541, 21.11248207092285, 1.0387442111968994], step: 8800, lr: 9.847416455282387e-05, reference_loss: 36.00392532348633
648
+ 2023-05-03 15:11:23,885 44k INFO Saving model and optimizer state at iteration 124 to ./logs/44k/G_8800.pth
649
+ 2023-05-03 15:11:25,575 44k INFO Saving model and optimizer state at iteration 124 to ./logs/44k/D_8800.pth
650
+ 2023-05-03 15:11:26,127 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8600.pth
651
+ 2023-05-03 15:11:26,179 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8600.pth
652
+ 2023-05-03 15:11:28,960 44k INFO ====> Epoch: 124, cost 54.32 s
653
+ 2023-05-03 15:12:15,260 44k INFO ====> Epoch: 125, cost 46.30 s
654
+ 2023-05-03 15:12:34,036 44k INFO Train Epoch: 126 [34%]
655
+ 2023-05-03 15:12:34,036 44k INFO Losses: [2.5524790287017822, 2.10302734375, 6.694677352905273, 18.45949935913086, 1.0567528009414673], step: 8900, lr: 9.84495475503445e-05, reference_loss: 30.866436004638672
656
+ 2023-05-03 15:12:39,212 44k INFO Saving model and optimizer state at iteration 126 to ./logs/44k/G_8900.pth
657
+ 2023-05-03 15:12:40,917 44k INFO Saving model and optimizer state at iteration 126 to ./logs/44k/D_8900.pth
658
+ 2023-05-03 15:12:41,479 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8700.pth
659
+ 2023-05-03 15:12:41,526 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8700.pth
660
+ 2023-05-03 15:13:09,584 44k INFO ====> Epoch: 126, cost 54.32 s
661
+ 2023-05-03 15:13:45,650 44k INFO Train Epoch: 127 [75%]
662
+ 2023-05-03 15:13:45,650 44k INFO Losses: [2.4279212951660156, 2.272848606109619, 7.698683738708496, 22.946094512939453, 1.227624535560608], step: 9000, lr: 9.84372413569007e-05, reference_loss: 36.57317352294922
663
+ 2023-05-03 15:13:50,682 44k INFO Saving model and optimizer state at iteration 127 to ./logs/44k/G_9000.pth
664
+ 2023-05-03 15:13:52,264 44k INFO Saving model and optimizer state at iteration 127 to ./logs/44k/D_9000.pth
665
+ 2023-05-03 15:13:52,825 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8800.pth
666
+ 2023-05-03 15:13:52,855 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8800.pth
667
+ 2023-05-03 15:14:03,357 44k INFO ====> Epoch: 127, cost 53.77 s
668
+ 2023-05-03 15:14:50,208 44k INFO ====> Epoch: 128, cost 46.85 s
669
+ 2023-05-03 15:15:01,752 44k INFO Train Epoch: 129 [15%]
670
+ 2023-05-03 15:15:01,753 44k INFO Losses: [2.2435436248779297, 2.89982271194458, 11.728398323059082, 20.402381896972656, 1.4633480310440063], step: 9100, lr: 9.841263358464336e-05, reference_loss: 38.73749542236328
671
+ 2023-05-03 15:15:07,343 44k INFO Saving model and optimizer state at iteration 129 to ./logs/44k/G_9100.pth
672
+ 2023-05-03 15:15:09,302 44k INFO Saving model and optimizer state at iteration 129 to ./logs/44k/D_9100.pth
673
+ 2023-05-03 15:15:09,957 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8900.pth
674
+ 2023-05-03 15:15:09,957 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8900.pth
675
+ 2023-05-03 15:15:45,576 44k INFO ====> Epoch: 129, cost 55.37 s
676
+ 2023-05-03 15:16:13,902 44k INFO Train Epoch: 130 [56%]
677
+ 2023-05-03 15:16:13,903 44k INFO Losses: [2.409078359603882, 2.190520763397217, 8.512211799621582, 19.713029861450195, 1.1344952583312988], step: 9200, lr: 9.840033200544528e-05, reference_loss: 33.95933532714844
678
+ 2023-05-03 15:16:19,095 44k INFO Saving model and optimizer state at iteration 130 to ./logs/44k/G_9200.pth
679
+ 2023-05-03 15:16:20,756 44k INFO Saving model and optimizer state at iteration 130 to ./logs/44k/D_9200.pth
680
+ 2023-05-03 15:16:21,317 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9000.pth
681
+ 2023-05-03 15:16:21,365 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9000.pth
682
+ 2023-05-03 15:16:39,759 44k INFO ====> Epoch: 130, cost 54.18 s
683
+ 2023-05-03 15:17:25,441 44k INFO Train Epoch: 131 [97%]
684
+ 2023-05-03 15:17:25,442 44k INFO Losses: [2.3736002445220947, 2.522505283355713, 9.207528114318848, 20.819185256958008, 1.211393117904663], step: 9300, lr: 9.838803196394459e-05, reference_loss: 36.134212493896484
685
+ 2023-05-03 15:17:30,582 44k INFO Saving model and optimizer state at iteration 131 to ./logs/44k/G_9300.pth
686
+ 2023-05-03 15:17:32,333 44k INFO Saving model and optimizer state at iteration 131 to ./logs/44k/D_9300.pth
687
+ 2023-05-03 15:17:32,921 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9100.pth
688
+ 2023-05-03 15:17:32,973 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9100.pth
689
+ 2023-05-03 15:17:34,031 44k INFO ====> Epoch: 131, cost 54.27 s
690
+ 2023-05-03 15:18:20,663 44k INFO ====> Epoch: 132, cost 46.63 s
691
+ 2023-05-03 15:18:41,162 44k INFO Train Epoch: 133 [38%]
692
+ 2023-05-03 15:18:41,162 44k INFO Losses: [2.4086642265319824, 2.2884583473205566, 7.530864715576172, 20.83812141418457, 1.2712756395339966], step: 9400, lr: 9.836343649326659e-05, reference_loss: 34.33738327026367
693
+ 2023-05-03 15:18:46,365 44k INFO Saving model and optimizer state at iteration 133 to ./logs/44k/G_9400.pth
694
+ 2023-05-03 15:18:48,073 44k INFO Saving model and optimizer state at iteration 133 to ./logs/44k/D_9400.pth
695
+ 2023-05-03 15:18:48,642 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9200.pth
696
+ 2023-05-03 15:18:48,694 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9200.pth
697
+ 2023-05-03 15:19:14,844 44k INFO ====> Epoch: 133, cost 54.18 s
698
+ 2023-05-03 15:19:53,005 44k INFO Train Epoch: 134 [79%]
699
+ 2023-05-03 15:19:53,006 44k INFO Losses: [2.2663352489471436, 2.593853235244751, 9.784102439880371, 20.418903350830078, 1.4837355613708496], step: 9500, lr: 9.835114106370493e-05, reference_loss: 36.54692840576172
700
+ 2023-05-03 15:19:58,125 44k INFO Saving model and optimizer state at iteration 134 to ./logs/44k/G_9500.pth
701
+ 2023-05-03 15:19:59,861 44k INFO Saving model and optimizer state at iteration 134 to ./logs/44k/D_9500.pth
702
+ 2023-05-03 15:20:00,423 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9300.pth
703
+ 2023-05-03 15:20:00,476 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9300.pth
704
+ 2023-05-03 15:20:09,180 44k INFO ====> Epoch: 134, cost 54.34 s
705
+ 2023-05-03 15:20:56,127 44k INFO ====> Epoch: 135, cost 46.95 s
706
+ 2023-05-03 15:21:08,711 44k INFO Train Epoch: 136 [20%]
707
+ 2023-05-03 15:21:08,711 44k INFO Losses: [2.484848976135254, 2.1545777320861816, 12.452383995056152, 19.72847557067871, 1.328551173210144], step: 9600, lr: 9.832655481517557e-05, reference_loss: 38.14883804321289
708
+ 2023-05-03 15:21:13,958 44k INFO Saving model and optimizer state at iteration 136 to ./logs/44k/G_9600.pth
709
+ 2023-05-03 15:21:15,541 44k INFO Saving model and optimizer state at iteration 136 to ./logs/44k/D_9600.pth
710
+ 2023-05-03 15:21:16,118 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9400.pth
711
+ 2023-05-03 15:21:16,170 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9400.pth
712
+ 2023-05-03 15:21:49,828 44k INFO ====> Epoch: 136, cost 53.70 s
713
+ 2023-05-03 15:22:19,843 44k INFO Train Epoch: 137 [61%]
714
+ 2023-05-03 15:22:19,844 44k INFO Losses: [2.525221824645996, 2.226086378097534, 9.078721046447754, 22.079233169555664, 0.9952948093414307], step: 9700, lr: 9.831426399582366e-05, reference_loss: 36.90456008911133
715
+ 2023-05-03 15:22:24,949 44k INFO Saving model and optimizer state at iteration 137 to ./logs/44k/G_9700.pth
716
+ 2023-05-03 15:22:26,787 44k INFO Saving model and optimizer state at iteration 137 to ./logs/44k/D_9700.pth
717
+ 2023-05-03 15:22:27,360 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9500.pth
718
+ 2023-05-03 15:22:27,412 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9500.pth
719
+ 2023-05-03 15:22:43,723 44k INFO ====> Epoch: 137, cost 53.89 s
720
+ 2023-05-03 15:23:30,200 44k INFO ====> Epoch: 138, cost 46.48 s
721
+ 2023-05-03 15:23:34,926 44k INFO Train Epoch: 139 [1%]
722
+ 2023-05-03 15:23:34,926 44k INFO Losses: [2.4429895877838135, 2.154385805130005, 8.6116943359375, 20.738656997680664, 1.326363205909729], step: 9800, lr: 9.828968696598508e-05, reference_loss: 35.27408981323242
723
+ 2023-05-03 15:23:39,999 44k INFO Saving model and optimizer state at iteration 139 to ./logs/44k/G_9800.pth
724
+ 2023-05-03 15:23:41,610 44k INFO Saving model and optimizer state at iteration 139 to ./logs/44k/D_9800.pth
725
+ 2023-05-03 15:23:42,172 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9600.pth
726
+ 2023-05-03 15:23:42,225 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9600.pth
727
+ 2023-05-03 15:24:23,989 44k INFO ====> Epoch: 139, cost 53.79 s
728
+ 2023-05-03 15:24:46,199 44k INFO Train Epoch: 140 [42%]
729
+ 2023-05-03 15:24:46,200 44k INFO Losses: [2.2960493564605713, 2.411436080932617, 8.397132873535156, 22.00579833984375, 1.3239285945892334], step: 9900, lr: 9.827740075511432e-05, reference_loss: 36.43434524536133
730
+ 2023-05-03 15:24:51,236 44k INFO Saving model and optimizer state at iteration 140 to ./logs/44k/G_9900.pth
731
+ 2023-05-03 15:24:52,957 44k INFO Saving model and optimizer state at iteration 140 to ./logs/44k/D_9900.pth
732
+ 2023-05-03 15:24:53,546 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9700.pth
733
+ 2023-05-03 15:24:53,600 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9700.pth
734
+ 2023-05-03 15:25:18,300 44k INFO ====> Epoch: 140, cost 54.31 s
735
+ 2023-05-03 15:25:58,385 44k INFO Train Epoch: 141 [83%]
736
+ 2023-05-03 15:25:58,385 44k INFO Losses: [2.2452259063720703, 2.455890655517578, 9.378299713134766, 23.54975128173828, 1.2288228273391724], step: 10000, lr: 9.826511608001993e-05, reference_loss: 38.85799026489258
737
+ 2023-05-03 15:26:03,556 44k INFO Saving model and optimizer state at iteration 141 to ./logs/44k/G_10000.pth
738
+ 2023-05-03 15:26:05,260 44k INFO Saving model and optimizer state at iteration 141 to ./logs/44k/D_10000.pth
739
+ 2023-05-03 15:26:05,837 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9800.pth
740
+ 2023-05-03 15:26:05,890 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9800.pth
741
+ 2023-05-03 15:26:12,950 44k INFO ====> Epoch: 141, cost 54.65 s
742
+ 2023-05-03 15:26:59,982 44k INFO ====> Epoch: 142, cost 47.03 s
743
+ 2023-05-03 15:27:14,406 44k INFO Train Epoch: 143 [24%]
744
+ 2023-05-03 15:27:14,407 44k INFO Losses: [2.392941951751709, 2.2666730880737305, 7.481887340545654, 21.5576171875, 1.1281098127365112], step: 10100, lr: 9.824055133639235e-05, reference_loss: 34.82722854614258
745
+ 2023-05-03 15:27:19,475 44k INFO Saving model and optimizer state at iteration 143 to ./logs/44k/G_10100.pth
746
+ 2023-05-03 15:27:21,296 44k INFO Saving model and optimizer state at iteration 143 to ./logs/44k/D_10100.pth
747
+ 2023-05-03 15:27:21,896 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9900.pth
748
+ 2023-05-03 15:27:21,948 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9900.pth
749
+ 2023-05-03 15:27:53,922 44k INFO ====> Epoch: 143, cost 53.94 s
750
+ 2023-05-03 15:28:26,261 44k INFO Train Epoch: 144 [65%]
751
+ 2023-05-03 15:28:26,262 44k INFO Losses: [2.138638973236084, 2.2958712577819824, 12.25405502319336, 19.357269287109375, 0.7592658996582031], step: 10200, lr: 9.822827126747529e-05, reference_loss: 36.80509948730469
752
+ 2023-05-03 15:28:31,318 44k INFO Saving model and optimizer state at iteration 144 to ./logs/44k/G_10200.pth
753
+ 2023-05-03 15:28:33,087 44k INFO Saving model and optimizer state at iteration 144 to ./logs/44k/D_10200.pth
754
+ 2023-05-03 15:28:33,666 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10000.pth
755
+ 2023-05-03 15:28:33,718 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10000.pth
756
+ 2023-05-03 15:28:48,562 44k INFO ====> Epoch: 144, cost 54.64 s
757
+ 2023-05-03 15:29:35,378 44k INFO ====> Epoch: 145, cost 46.82 s
758
+ 2023-05-03 15:29:42,058 44k INFO Train Epoch: 146 [6%]
759
+ 2023-05-03 15:29:42,059 44k INFO Losses: [2.4405500888824463, 2.3585987091064453, 7.771078586578369, 21.579200744628906, 1.055567979812622], step: 10300, lr: 9.820371573447515e-05, reference_loss: 35.20499801635742
760
+ 2023-05-03 15:29:47,198 44k INFO Saving model and optimizer state at iteration 146 to ./logs/44k/G_10300.pth
761
+ 2023-05-03 15:29:48,895 44k INFO Saving model and optimizer state at iteration 146 to ./logs/44k/D_10300.pth
762
+ 2023-05-03 15:29:49,470 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10100.pth
763
+ 2023-05-03 15:29:49,522 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10100.pth
764
+ 2023-05-03 15:30:29,636 44k INFO ====> Epoch: 146, cost 54.26 s
765
+ 2023-05-03 15:30:53,921 44k INFO Train Epoch: 147 [46%]
766
+ 2023-05-03 15:30:53,922 44k INFO Losses: [2.4801368713378906, 2.6744041442871094, 10.43703556060791, 19.093036651611328, 0.8212056159973145], step: 10400, lr: 9.819144027000834e-05, reference_loss: 35.50581741333008
767
+ 2023-05-03 15:30:59,066 44k INFO Saving model and optimizer state at iteration 147 to ./logs/44k/G_10400.pth
768
+ 2023-05-03 15:31:00,779 44k INFO Saving model and optimizer state at iteration 147 to ./logs/44k/D_10400.pth
769
+ 2023-05-03 15:31:01,355 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10200.pth
770
+ 2023-05-03 15:31:01,407 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10200.pth
771
+ 2023-05-03 15:31:24,078 44k INFO ====> Epoch: 147, cost 54.44 s
772
+ 2023-05-03 15:32:10,012 44k INFO Train Epoch: 148 [87%]
773
+ 2023-05-03 15:32:10,013 44k INFO Losses: [2.3031187057495117, 2.331038236618042, 12.805354118347168, 21.40829849243164, 0.8880621194839478], step: 10500, lr: 9.817916633997459e-05, reference_loss: 39.735870361328125
774
+ 2023-05-03 15:32:15,130 44k INFO Saving model and optimizer state at iteration 148 to ./logs/44k/G_10500.pth
775
+ 2023-05-03 15:32:16,899 44k INFO Saving model and optimizer state at iteration 148 to ./logs/44k/D_10500.pth
776
+ 2023-05-03 15:32:17,477 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10300.pth
777
+ 2023-05-03 15:32:17,529 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10300.pth
778
+ 2023-05-03 15:32:22,759 44k INFO ====> Epoch: 148, cost 58.68 s
779
+ 2023-05-03 15:33:09,200 44k INFO ====> Epoch: 149, cost 46.44 s
780
+ 2023-05-03 15:33:25,713 44k INFO Train Epoch: 150 [28%]
781
+ 2023-05-03 15:33:25,714 44k INFO Losses: [2.7253715991973877, 2.3599607944488525, 5.7014360427856445, 17.386552810668945, 1.0759114027023315], step: 10600, lr: 9.815462308243906e-05, reference_loss: 29.249231338500977
782
+ 2023-05-03 15:33:30,700 44k INFO Saving model and optimizer state at iteration 150 to ./logs/44k/G_10600.pth
783
+ 2023-05-03 15:33:32,395 44k INFO Saving model and optimizer state at iteration 150 to ./logs/44k/D_10600.pth
784
+ 2023-05-03 15:33:32,964 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10400.pth
785
+ 2023-05-03 15:33:33,016 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10400.pth
786
+ 2023-05-03 15:34:03,304 44k INFO ====> Epoch: 150, cost 54.10 s
787
+ 2023-05-03 15:34:37,855 44k INFO Train Epoch: 151 [69%]
788
+ 2023-05-03 15:34:37,855 44k INFO Losses: [2.3957998752593994, 2.5786211490631104, 8.307215690612793, 20.157119750976562, 0.9237015843391418], step: 10700, lr: 9.814235375455375e-05, reference_loss: 34.36246109008789
789
+ 2023-05-03 15:34:43,152 44k INFO Saving model and optimizer state at iteration 151 to ./logs/44k/G_10700.pth
790
+ 2023-05-03 15:34:44,852 44k INFO Saving model and optimizer state at iteration 151 to ./logs/44k/D_10700.pth
791
+ 2023-05-03 15:34:45,424 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10500.pth
792
+ 2023-05-03 15:34:45,476 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10500.pth
793
+ 2023-05-03 15:34:58,329 44k INFO ====> Epoch: 151, cost 55.02 s
794
+ 2023-05-03 15:35:45,281 44k INFO ====> Epoch: 152, cost 46.95 s
795
+ 2023-05-03 15:35:53,820 44k INFO Train Epoch: 153 [10%]
796
+ 2023-05-03 15:35:53,821 44k INFO Losses: [2.449925422668457, 2.4277830123901367, 6.188196659088135, 20.815114974975586, 0.7682157158851624], step: 10800, lr: 9.811781969958938e-05, reference_loss: 32.649234771728516
797
+ 2023-05-03 15:35:59,223 44k INFO Saving model and optimizer state at iteration 153 to ./logs/44k/G_10800.pth
798
+ 2023-05-03 15:36:00,869 44k INFO Saving model and optimizer state at iteration 153 to ./logs/44k/D_10800.pth
799
+ 2023-05-03 15:36:01,450 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10600.pth
800
+ 2023-05-03 15:36:01,503 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10600.pth
801
+ 2023-05-03 15:36:40,018 44k INFO ====> Epoch: 153, cost 54.74 s
802
+ 2023-05-03 15:37:05,909 44k INFO Train Epoch: 154 [51%]
803
+ 2023-05-03 15:37:05,910 44k INFO Losses: [2.2990469932556152, 2.5312039852142334, 8.215372085571289, 19.852989196777344, 1.0274193286895752], step: 10900, lr: 9.810555497212693e-05, reference_loss: 33.92603302001953
804
+ 2023-05-03 15:37:10,984 44k INFO Saving model and optimizer state at iteration 154 to ./logs/44k/G_10900.pth
805
+ 2023-05-03 15:37:12,868 44k INFO Saving model and optimizer state at iteration 154 to ./logs/44k/D_10900.pth
806
+ 2023-05-03 15:37:13,446 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10700.pth
807
+ 2023-05-03 15:37:13,498 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10700.pth
808
+ 2023-05-03 15:37:34,031 44k INFO ====> Epoch: 154, cost 54.01 s
809
+ 2023-05-03 15:38:17,770 44k INFO Train Epoch: 155 [92%]
810
+ 2023-05-03 15:38:17,771 44k INFO Losses: [2.347559928894043, 2.624699592590332, 10.70503044128418, 18.965770721435547, 0.6355746984481812], step: 11000, lr: 9.809329177775541e-05, reference_loss: 35.27863311767578
811
+ 2023-05-03 15:38:23,015 44k INFO Saving model and optimizer state at iteration 155 to ./logs/44k/G_11000.pth
812
+ 2023-05-03 15:38:24,698 44k INFO Saving model and optimizer state at iteration 155 to ./logs/44k/D_11000.pth
813
+ 2023-05-03 15:38:25,283 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10800.pth
814
+ 2023-05-03 15:38:25,336 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10800.pth
815
+ 2023-05-03 15:38:28,805 44k INFO ====> Epoch: 155, cost 54.77 s
816
+ 2023-05-03 15:39:15,570 44k INFO ====> Epoch: 156, cost 46.76 s
817
+ 2023-05-03 15:39:35,466 44k INFO Train Epoch: 157 [32%]
818
+ 2023-05-03 15:39:35,467 44k INFO Losses: [2.576101303100586, 2.190938949584961, 7.242859840393066, 21.8758602142334, 0.8499348163604736], step: 11100, lr: 9.806876998751865e-05, reference_loss: 34.735694885253906
819
+ 2023-05-03 15:39:40,781 44k INFO Saving model and optimizer state at iteration 157 to ./logs/44k/G_11100.pth
820
+ 2023-05-03 15:39:42,545 44k INFO Saving model and optimizer state at iteration 157 to ./logs/44k/D_11100.pth
821
+ 2023-05-03 15:39:43,126 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10900.pth
822
+ 2023-05-03 15:39:43,178 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10900.pth
823
+ 2023-05-03 15:40:13,774 44k INFO ====> Epoch: 157, cost 58.20 s
824
+ 2023-05-03 15:40:49,393 44k INFO Train Epoch: 158 [73%]
825
+ 2023-05-03 15:40:49,394 44k INFO Losses: [2.4354920387268066, 2.526336193084717, 9.373090744018555, 20.170114517211914, 1.5362391471862793], step: 11200, lr: 9.80565113912702e-05, reference_loss: 36.04127502441406
826
+ 2023-05-03 15:40:54,411 44k INFO Saving model and optimizer state at iteration 158 to ./logs/44k/G_11200.pth
827
+ 2023-05-03 15:40:56,229 44k INFO Saving model and optimizer state at iteration 158 to ./logs/44k/D_11200.pth
828
+ 2023-05-03 15:40:56,811 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11000.pth
829
+ 2023-05-03 15:40:56,863 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11000.pth
830
+ 2023-05-03 15:41:07,949 44k INFO ====> Epoch: 158, cost 54.17 s
831
+ 2023-05-03 15:41:55,183 44k INFO ====> Epoch: 159, cost 47.23 s
832
+ 2023-05-03 15:42:05,496 44k INFO Train Epoch: 160 [14%]
833
+ 2023-05-03 15:42:05,497 44k INFO Losses: [2.44669246673584, 2.414104461669922, 9.693013191223145, 20.203325271606445, 1.273841381072998], step: 11300, lr: 9.803199879555537e-05, reference_loss: 36.030975341796875
834
+ 2023-05-03 15:42:10,663 44k INFO Saving model and optimizer state at iteration 160 to ./logs/44k/G_11300.pth
835
+ 2023-05-03 15:42:12,281 44k INFO Saving model and optimizer state at iteration 160 to ./logs/44k/D_11300.pth
836
+ 2023-05-03 15:42:12,863 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11100.pth
837
+ 2023-05-03 15:42:12,915 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11100.pth
838
+ 2023-05-03 15:42:49,118 44k INFO ====> Epoch: 160, cost 53.93 s
839
+ 2023-05-03 15:43:16,997 44k INFO Train Epoch: 161 [55%]
840
+ 2023-05-03 15:43:16,997 44k INFO Losses: [2.445122241973877, 2.363597869873047, 10.572999954223633, 20.042144775390625, 1.0533027648925781], step: 11400, lr: 9.801974479570593e-05, reference_loss: 36.477169036865234
841
+ 2023-05-03 15:43:22,411 44k INFO Saving model and optimizer state at iteration 161 to ./logs/44k/G_11400.pth
842
+ 2023-05-03 15:43:24,192 44k INFO Saving model and optimizer state at iteration 161 to ./logs/44k/D_11400.pth
843
+ 2023-05-03 15:43:24,775 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11200.pth
844
+ 2023-05-03 15:43:24,828 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11200.pth
845
+ 2023-05-03 15:43:45,908 44k INFO ====> Epoch: 161, cost 56.79 s
846
+ 2023-05-03 15:44:31,322 44k INFO Train Epoch: 162 [96%]
847
+ 2023-05-03 15:44:31,323 44k INFO Losses: [2.3155155181884766, 2.432081699371338, 10.127751350402832, 20.838157653808594, 0.8625903725624084], step: 11500, lr: 9.800749232760646e-05, reference_loss: 36.57609939575195
848
+ 2023-05-03 15:44:36,426 44k INFO Saving model and optimizer state at iteration 162 to ./logs/44k/G_11500.pth
849
+ 2023-05-03 15:44:38,226 44k INFO Saving model and optimizer state at iteration 162 to ./logs/44k/D_11500.pth
850
+ 2023-05-03 15:44:38,806 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11300.pth
851
+ 2023-05-03 15:44:38,858 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11300.pth
852
+ 2023-05-03 15:44:40,491 44k INFO ====> Epoch: 162, cost 54.58 s