add files
Browse files- D_10600.pth +3 -0
- D_11500.pth +3 -0
- G_10600.pth +3 -0
- G_11500.pth +3 -0
- config.json +94 -0
- eval/events.out.tfevents.1683120118.PY4ddV.774.1 +3 -0
- events.out.tfevents.1683120118.PY4ddV.774.0 +3 -0
- train.log +852 -0
D_10600.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f95f94ba4337d8184aa6d828f5ac1e2f74346ea8c651c67d1af65133a6c069c3
|
3 |
+
size 561098185
|
D_11500.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:cadcb0a03397ff5bf528afbc73ad31c2859f3f14886346db5f8ac3209ad5388e
|
3 |
+
size 561098185
|
G_10600.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f6af4253cb85eb6f6bdea557098501f54942da8ba5533606ef26427ace7478f0
|
3 |
+
size 542789405
|
G_11500.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f4defb1685aadaf969dd438efe55acd168c61f1db485b77981099023fa02cdf5
|
3 |
+
size 542789405
|
config.json
ADDED
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"train": {
|
3 |
+
"log_interval": 100,
|
4 |
+
"eval_interval": 100,
|
5 |
+
"seed": 1234,
|
6 |
+
"epochs": 800,
|
7 |
+
"learning_rate": 0.0001,
|
8 |
+
"betas": [
|
9 |
+
0.8,
|
10 |
+
0.99
|
11 |
+
],
|
12 |
+
"eps": 1e-09,
|
13 |
+
"batch_size": 6,
|
14 |
+
"fp16_run": false,
|
15 |
+
"lr_decay": 0.999875,
|
16 |
+
"segment_size": 10240,
|
17 |
+
"init_lr_ratio": 1,
|
18 |
+
"warmup_epochs": 0,
|
19 |
+
"c_mel": 45,
|
20 |
+
"c_kl": 1.0,
|
21 |
+
"use_sr": true,
|
22 |
+
"max_speclen": 512,
|
23 |
+
"port": "8001",
|
24 |
+
"keep_ckpts": 2,
|
25 |
+
"all_in_mem": false
|
26 |
+
},
|
27 |
+
"data": {
|
28 |
+
"training_files": "filelists/train.txt",
|
29 |
+
"validation_files": "filelists/val.txt",
|
30 |
+
"max_wav_value": 32768.0,
|
31 |
+
"sampling_rate": 44100,
|
32 |
+
"filter_length": 2048,
|
33 |
+
"hop_length": 512,
|
34 |
+
"win_length": 2048,
|
35 |
+
"n_mel_channels": 80,
|
36 |
+
"mel_fmin": 0.0,
|
37 |
+
"mel_fmax": 22050
|
38 |
+
},
|
39 |
+
"model": {
|
40 |
+
"inter_channels": 192,
|
41 |
+
"hidden_channels": 192,
|
42 |
+
"filter_channels": 768,
|
43 |
+
"n_heads": 2,
|
44 |
+
"n_layers": 6,
|
45 |
+
"kernel_size": 3,
|
46 |
+
"p_dropout": 0.1,
|
47 |
+
"resblock": "1",
|
48 |
+
"resblock_kernel_sizes": [
|
49 |
+
3,
|
50 |
+
7,
|
51 |
+
11
|
52 |
+
],
|
53 |
+
"resblock_dilation_sizes": [
|
54 |
+
[
|
55 |
+
1,
|
56 |
+
3,
|
57 |
+
5
|
58 |
+
],
|
59 |
+
[
|
60 |
+
1,
|
61 |
+
3,
|
62 |
+
5
|
63 |
+
],
|
64 |
+
[
|
65 |
+
1,
|
66 |
+
3,
|
67 |
+
5
|
68 |
+
]
|
69 |
+
],
|
70 |
+
"upsample_rates": [
|
71 |
+
8,
|
72 |
+
8,
|
73 |
+
2,
|
74 |
+
2,
|
75 |
+
2
|
76 |
+
],
|
77 |
+
"upsample_initial_channel": 512,
|
78 |
+
"upsample_kernel_sizes": [
|
79 |
+
16,
|
80 |
+
16,
|
81 |
+
4,
|
82 |
+
4,
|
83 |
+
4
|
84 |
+
],
|
85 |
+
"n_layers_q": 3,
|
86 |
+
"use_spectral_norm": false,
|
87 |
+
"gin_channels": 256,
|
88 |
+
"ssl_dim": 256,
|
89 |
+
"n_speakers": 200
|
90 |
+
},
|
91 |
+
"spk": {
|
92 |
+
"danxiao": 0
|
93 |
+
}
|
94 |
+
}
|
eval/events.out.tfevents.1683120118.PY4ddV.774.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2f938061d4f31445606faafd1737255c66e9524b5ba5bbb6faf4f1687cb10751
|
3 |
+
size 345535863
|
events.out.tfevents.1683120118.PY4ddV.774.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2cb2c2ffc50ce28ddb00f019a8ac4f95711b47e1784003aabb145d2282e9f6eb
|
3 |
+
size 22616873
|
train.log
ADDED
@@ -0,0 +1,852 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2023-05-03 13:21:58,872 44k INFO {'train': {'log_interval': 100, 'eval_interval': 100, 'seed': 1234, 'epochs': 800, 'learning_rate': 0.0001, 'betas': [0.8, 0.99], 'eps': 1e-09, 'batch_size': 6, 'fp16_run': False, 'lr_decay': 0.999875, 'segment_size': 10240, 'init_lr_ratio': 1, 'warmup_epochs': 0, 'c_mel': 45, 'c_kl': 1.0, 'use_sr': True, 'max_speclen': 512, 'port': '8001', 'keep_ckpts': 2, 'all_in_mem': False}, 'data': {'training_files': 'filelists/train.txt', 'validation_files': 'filelists/val.txt', 'max_wav_value': 32768.0, 'sampling_rate': 44100, 'filter_length': 2048, 'hop_length': 512, 'win_length': 2048, 'n_mel_channels': 80, 'mel_fmin': 0.0, 'mel_fmax': 22050}, 'model': {'inter_channels': 192, 'hidden_channels': 192, 'filter_channels': 768, 'n_heads': 2, 'n_layers': 6, 'kernel_size': 3, 'p_dropout': 0.1, 'resblock': '1', 'resblock_kernel_sizes': [3, 7, 11], 'resblock_dilation_sizes': [[1, 3, 5], [1, 3, 5], [1, 3, 5]], 'upsample_rates': [8, 8, 2, 2, 2], 'upsample_initial_channel': 512, 'upsample_kernel_sizes': [16, 16, 4, 4, 4], 'n_layers_q': 3, 'use_spectral_norm': False, 'gin_channels': 256, 'ssl_dim': 256, 'n_speakers': 200}, 'spk': {'danxiao': 0}, 'model_dir': './logs/44k'}
|
2 |
+
2023-05-03 13:22:01,597 44k INFO emb_g.weight is not in the checkpoint
|
3 |
+
2023-05-03 13:22:01,663 44k INFO Loaded checkpoint './logs/44k/G_0.pth' (iteration 0)
|
4 |
+
2023-05-03 13:22:01,753 44k INFO Loaded checkpoint './logs/44k/D_0.pth' (iteration 0)
|
5 |
+
2023-05-03 13:22:55,617 44k INFO ====> Epoch: 1, cost 56.75 s
|
6 |
+
2023-05-03 13:23:16,634 44k INFO Train Epoch: 2 [39%]
|
7 |
+
2023-05-03 13:23:16,635 44k INFO Losses: [2.0492827892303467, 2.659921646118164, 10.786680221557617, 31.86163330078125, 1.4900364875793457], step: 100, lr: 9.99875e-05, reference_loss: 48.84755325317383
|
8 |
+
2023-05-03 13:23:23,145 44k INFO Saving model and optimizer state at iteration 2 to ./logs/44k/G_100.pth
|
9 |
+
2023-05-03 13:23:24,867 44k INFO Saving model and optimizer state at iteration 2 to ./logs/44k/D_100.pth
|
10 |
+
2023-05-03 13:23:50,972 44k INFO ====> Epoch: 2, cost 55.35 s
|
11 |
+
2023-05-03 13:24:29,866 44k INFO Train Epoch: 3 [80%]
|
12 |
+
2023-05-03 13:24:29,867 44k INFO Losses: [2.5209755897521973, 2.313697576522827, 9.055400848388672, 24.980056762695312, 1.70906400680542], step: 200, lr: 9.99750015625e-05, reference_loss: 40.57919692993164
|
13 |
+
2023-05-03 13:24:34,955 44k INFO Saving model and optimizer state at iteration 3 to ./logs/44k/G_200.pth
|
14 |
+
2023-05-03 13:24:36,690 44k INFO Saving model and optimizer state at iteration 3 to ./logs/44k/D_200.pth
|
15 |
+
2023-05-03 13:24:45,486 44k INFO ====> Epoch: 3, cost 54.51 s
|
16 |
+
2023-05-03 13:25:32,621 44k INFO ====> Epoch: 4, cost 47.14 s
|
17 |
+
2023-05-03 13:25:46,099 44k INFO Train Epoch: 5 [21%]
|
18 |
+
2023-05-03 13:25:46,100 44k INFO Losses: [2.311826467514038, 2.4695792198181152, 10.9570894241333, 22.569786071777344, 1.274253249168396], step: 300, lr: 9.995000937421877e-05, reference_loss: 39.58253479003906
|
19 |
+
2023-05-03 13:25:51,325 44k INFO Saving model and optimizer state at iteration 5 to ./logs/44k/G_300.pth
|
20 |
+
2023-05-03 13:25:53,076 44k INFO Saving model and optimizer state at iteration 5 to ./logs/44k/D_300.pth
|
21 |
+
2023-05-03 13:25:53,605 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_100.pth
|
22 |
+
2023-05-03 13:25:53,634 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_100.pth
|
23 |
+
2023-05-03 13:26:27,242 44k INFO ====> Epoch: 5, cost 54.62 s
|
24 |
+
2023-05-03 13:26:59,132 44k INFO Train Epoch: 6 [62%]
|
25 |
+
2023-05-03 13:26:59,133 44k INFO Losses: [2.14005708694458, 2.4915013313293457, 10.79394817352295, 22.39518928527832, 1.7690892219543457], step: 400, lr: 9.993751562304699e-05, reference_loss: 39.58978271484375
|
26 |
+
2023-05-03 13:27:04,372 44k INFO Saving model and optimizer state at iteration 6 to ./logs/44k/G_400.pth
|
27 |
+
2023-05-03 13:27:06,083 44k INFO Saving model and optimizer state at iteration 6 to ./logs/44k/D_400.pth
|
28 |
+
2023-05-03 13:27:06,613 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_200.pth
|
29 |
+
2023-05-03 13:27:06,642 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_200.pth
|
30 |
+
2023-05-03 13:27:22,649 44k INFO ====> Epoch: 6, cost 55.41 s
|
31 |
+
2023-05-03 13:28:09,748 44k INFO ====> Epoch: 7, cost 47.10 s
|
32 |
+
2023-05-03 13:28:15,219 44k INFO Train Epoch: 8 [3%]
|
33 |
+
2023-05-03 13:28:15,219 44k INFO Losses: [2.4994237422943115, 2.2374916076660156, 10.99351692199707, 27.435150146484375, 1.3526171445846558], step: 500, lr: 9.991253280566489e-05, reference_loss: 44.51819610595703
|
34 |
+
2023-05-03 13:28:20,329 44k INFO Saving model and optimizer state at iteration 8 to ./logs/44k/G_500.pth
|
35 |
+
2023-05-03 13:28:22,212 44k INFO Saving model and optimizer state at iteration 8 to ./logs/44k/D_500.pth
|
36 |
+
2023-05-03 13:28:22,746 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_300.pth
|
37 |
+
2023-05-03 13:28:22,774 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_300.pth
|
38 |
+
2023-05-03 13:29:04,358 44k INFO ====> Epoch: 8, cost 54.61 s
|
39 |
+
2023-05-03 13:29:27,578 44k INFO Train Epoch: 9 [44%]
|
40 |
+
2023-05-03 13:29:27,579 44k INFO Losses: [2.4906890392303467, 2.554823637008667, 9.332549095153809, 23.424272537231445, 1.4919406175613403], step: 600, lr: 9.990004373906418e-05, reference_loss: 39.294273376464844
|
41 |
+
2023-05-03 13:29:32,778 44k INFO Saving model and optimizer state at iteration 9 to ./logs/44k/G_600.pth
|
42 |
+
2023-05-03 13:29:34,547 44k INFO Saving model and optimizer state at iteration 9 to ./logs/44k/D_600.pth
|
43 |
+
2023-05-03 13:29:35,074 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_400.pth
|
44 |
+
2023-05-03 13:29:35,102 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_400.pth
|
45 |
+
2023-05-03 13:29:59,010 44k INFO ====> Epoch: 9, cost 54.65 s
|
46 |
+
2023-05-03 13:30:39,940 44k INFO Train Epoch: 10 [85%]
|
47 |
+
2023-05-03 13:30:39,941 44k INFO Losses: [2.4156060218811035, 2.1082849502563477, 11.128122329711914, 24.051517486572266, 1.5151060819625854], step: 700, lr: 9.98875562335968e-05, reference_loss: 41.21863555908203
|
48 |
+
2023-05-03 13:30:45,107 44k INFO Saving model and optimizer state at iteration 10 to ./logs/44k/G_700.pth
|
49 |
+
2023-05-03 13:30:46,837 44k INFO Saving model and optimizer state at iteration 10 to ./logs/44k/D_700.pth
|
50 |
+
2023-05-03 13:30:47,368 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_500.pth
|
51 |
+
2023-05-03 13:30:47,397 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_500.pth
|
52 |
+
2023-05-03 13:30:53,795 44k INFO ====> Epoch: 10, cost 54.79 s
|
53 |
+
2023-05-03 13:31:40,858 44k INFO ====> Epoch: 11, cost 47.06 s
|
54 |
+
2023-05-03 13:31:57,432 44k INFO Train Epoch: 12 [25%]
|
55 |
+
2023-05-03 13:31:57,433 44k INFO Losses: [2.3472788333892822, 2.697023630142212, 9.64306354522705, 21.859516143798828, 1.6248269081115723], step: 800, lr: 9.986258590528146e-05, reference_loss: 38.17171096801758
|
56 |
+
2023-05-03 13:32:02,662 44k INFO Saving model and optimizer state at iteration 12 to ./logs/44k/G_800.pth
|
57 |
+
2023-05-03 13:32:04,334 44k INFO Saving model and optimizer state at iteration 12 to ./logs/44k/D_800.pth
|
58 |
+
2023-05-03 13:32:04,862 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_600.pth
|
59 |
+
2023-05-03 13:32:04,890 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_600.pth
|
60 |
+
2023-05-03 13:32:36,561 44k INFO ====> Epoch: 12, cost 55.70 s
|
61 |
+
2023-05-03 13:33:09,525 44k INFO Train Epoch: 13 [66%]
|
62 |
+
2023-05-03 13:33:09,526 44k INFO Losses: [2.51536226272583, 2.6379146575927734, 7.380555629730225, 22.201026916503906, 1.308340072631836], step: 900, lr: 9.98501030820433e-05, reference_loss: 36.04319763183594
|
63 |
+
2023-05-03 13:33:14,629 44k INFO Saving model and optimizer state at iteration 13 to ./logs/44k/G_900.pth
|
64 |
+
2023-05-03 13:33:16,494 44k INFO Saving model and optimizer state at iteration 13 to ./logs/44k/D_900.pth
|
65 |
+
2023-05-03 13:33:17,022 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_700.pth
|
66 |
+
2023-05-03 13:33:17,051 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_700.pth
|
67 |
+
2023-05-03 13:33:31,182 44k INFO ====> Epoch: 13, cost 54.62 s
|
68 |
+
2023-05-03 13:34:17,814 44k INFO ====> Epoch: 14, cost 46.63 s
|
69 |
+
2023-05-03 13:34:25,110 44k INFO Train Epoch: 15 [7%]
|
70 |
+
2023-05-03 13:34:25,111 44k INFO Losses: [2.690953493118286, 2.0789146423339844, 8.32895278930664, 21.45608139038086, 1.2996363639831543], step: 1000, lr: 9.982514211643064e-05, reference_loss: 35.85453796386719
|
71 |
+
2023-05-03 13:34:30,133 44k INFO Saving model and optimizer state at iteration 15 to ./logs/44k/G_1000.pth
|
72 |
+
2023-05-03 13:34:31,795 44k INFO Saving model and optimizer state at iteration 15 to ./logs/44k/D_1000.pth
|
73 |
+
2023-05-03 13:34:32,325 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_800.pth
|
74 |
+
2023-05-03 13:34:32,353 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_800.pth
|
75 |
+
2023-05-03 13:35:11,691 44k INFO ====> Epoch: 15, cost 53.88 s
|
76 |
+
2023-05-03 13:35:39,789 44k INFO Train Epoch: 16 [48%]
|
77 |
+
2023-05-03 13:35:39,790 44k INFO Losses: [2.3972558975219727, 2.426987409591675, 8.578001022338867, 24.267925262451172, 1.3725520372390747], step: 1100, lr: 9.981266397366609e-05, reference_loss: 39.042720794677734
|
78 |
+
2023-05-03 13:35:44,968 44k INFO Saving model and optimizer state at iteration 16 to ./logs/44k/G_1100.pth
|
79 |
+
2023-05-03 13:35:46,747 44k INFO Saving model and optimizer state at iteration 16 to ./logs/44k/D_1100.pth
|
80 |
+
2023-05-03 13:35:47,285 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_900.pth
|
81 |
+
2023-05-03 13:35:47,313 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_900.pth
|
82 |
+
2023-05-03 13:36:10,496 44k INFO ====> Epoch: 16, cost 58.81 s
|
83 |
+
2023-05-03 13:36:57,589 44k INFO Train Epoch: 17 [89%]
|
84 |
+
2023-05-03 13:36:57,590 44k INFO Losses: [2.3657584190368652, 2.3757505416870117, 8.66689395904541, 22.137908935546875, 1.4960813522338867], step: 1200, lr: 9.980018739066937e-05, reference_loss: 37.04239273071289
|
85 |
+
2023-05-03 13:37:02,822 44k INFO Saving model and optimizer state at iteration 17 to ./logs/44k/G_1200.pth
|
86 |
+
2023-05-03 13:37:04,581 44k INFO Saving model and optimizer state at iteration 17 to ./logs/44k/D_1200.pth
|
87 |
+
2023-05-03 13:37:05,121 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1000.pth
|
88 |
+
2023-05-03 13:37:05,150 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1000.pth
|
89 |
+
2023-05-03 13:37:10,222 44k INFO ====> Epoch: 17, cost 59.73 s
|
90 |
+
2023-05-03 13:37:56,991 44k INFO ====> Epoch: 18, cost 46.77 s
|
91 |
+
2023-05-03 13:38:13,880 44k INFO Train Epoch: 19 [30%]
|
92 |
+
2023-05-03 13:38:13,881 44k INFO Losses: [2.3961024284362793, 2.4425392150878906, 10.046466827392578, 23.261734008789062, 1.3326858282089233], step: 1300, lr: 9.977523890319963e-05, reference_loss: 39.479530334472656
|
93 |
+
2023-05-03 13:38:18,930 44k INFO Saving model and optimizer state at iteration 19 to ./logs/44k/G_1300.pth
|
94 |
+
2023-05-03 13:38:20,769 44k INFO Saving model and optimizer state at iteration 19 to ./logs/44k/D_1300.pth
|
95 |
+
2023-05-03 13:38:21,295 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1100.pth
|
96 |
+
2023-05-03 13:38:21,323 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1100.pth
|
97 |
+
2023-05-03 13:38:51,063 44k INFO ====> Epoch: 19, cost 54.07 s
|
98 |
+
2023-05-03 13:39:26,290 44k INFO Train Epoch: 20 [70%]
|
99 |
+
2023-05-03 13:39:26,290 44k INFO Losses: [2.4467904567718506, 2.1601767539978027, 9.989028930664062, 21.93836784362793, 1.6368579864501953], step: 1400, lr: 9.976276699833672e-05, reference_loss: 38.17121887207031
|
100 |
+
2023-05-03 13:39:31,241 44k INFO Saving model and optimizer state at iteration 20 to ./logs/44k/G_1400.pth
|
101 |
+
2023-05-03 13:39:32,988 44k INFO Saving model and optimizer state at iteration 20 to ./logs/44k/D_1400.pth
|
102 |
+
2023-05-03 13:39:33,541 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1200.pth
|
103 |
+
2023-05-03 13:39:33,570 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1200.pth
|
104 |
+
2023-05-03 13:39:46,142 44k INFO ====> Epoch: 20, cost 55.08 s
|
105 |
+
2023-05-03 13:40:33,445 44k INFO ====> Epoch: 21, cost 47.30 s
|
106 |
+
2023-05-03 13:40:42,585 44k INFO Train Epoch: 22 [11%]
|
107 |
+
2023-05-03 13:40:42,586 44k INFO Losses: [2.4825923442840576, 1.9403694868087769, 9.403828620910645, 24.17758560180664, 1.701635718345642], step: 1500, lr: 9.973782786538036e-05, reference_loss: 39.70601272583008
|
108 |
+
2023-05-03 13:40:47,668 44k INFO Saving model and optimizer state at iteration 22 to ./logs/44k/G_1500.pth
|
109 |
+
2023-05-03 13:40:49,405 44k INFO Saving model and optimizer state at iteration 22 to ./logs/44k/D_1500.pth
|
110 |
+
2023-05-03 13:40:49,943 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1300.pth
|
111 |
+
2023-05-03 13:40:49,972 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1300.pth
|
112 |
+
2023-05-03 13:41:27,922 44k INFO ====> Epoch: 22, cost 54.48 s
|
113 |
+
2023-05-03 13:41:54,792 44k INFO Train Epoch: 23 [52%]
|
114 |
+
2023-05-03 13:41:54,793 44k INFO Losses: [2.400172472000122, 2.8658950328826904, 10.408707618713379, 25.947738647460938, 0.9147400856018066], step: 1600, lr: 9.972536063689719e-05, reference_loss: 42.537254333496094
|
115 |
+
2023-05-03 13:42:00,055 44k INFO Saving model and optimizer state at iteration 23 to ./logs/44k/G_1600.pth
|
116 |
+
2023-05-03 13:42:01,769 44k INFO Saving model and optimizer state at iteration 23 to ./logs/44k/D_1600.pth
|
117 |
+
2023-05-03 13:42:02,304 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1400.pth
|
118 |
+
2023-05-03 13:42:02,332 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1400.pth
|
119 |
+
2023-05-03 13:42:22,651 44k INFO ====> Epoch: 23, cost 54.73 s
|
120 |
+
2023-05-03 13:43:11,447 44k INFO Train Epoch: 24 [93%]
|
121 |
+
2023-05-03 13:43:11,448 44k INFO Losses: [2.320039749145508, 2.4628827571868896, 8.813612937927246, 21.163532257080078, 1.414304494857788], step: 1700, lr: 9.971289496681757e-05, reference_loss: 36.17436981201172
|
122 |
+
2023-05-03 13:43:16,627 44k INFO Saving model and optimizer state at iteration 24 to ./logs/44k/G_1700.pth
|
123 |
+
2023-05-03 13:43:18,476 44k INFO Saving model and optimizer state at iteration 24 to ./logs/44k/D_1700.pth
|
124 |
+
2023-05-03 13:43:19,015 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1500.pth
|
125 |
+
2023-05-03 13:43:19,043 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1500.pth
|
126 |
+
2023-05-03 13:43:22,120 44k INFO ====> Epoch: 24, cost 59.47 s
|
127 |
+
2023-05-03 13:44:08,810 44k INFO ====> Epoch: 25, cost 46.69 s
|
128 |
+
2023-05-03 13:44:28,358 44k INFO Train Epoch: 26 [34%]
|
129 |
+
2023-05-03 13:44:28,359 44k INFO Losses: [2.3216712474823, 2.4774258136749268, 9.157946586608887, 20.46706199645996, 1.4309511184692383], step: 1800, lr: 9.968796830108985e-05, reference_loss: 35.85505676269531
|
130 |
+
2023-05-03 13:44:33,579 44k INFO Saving model and optimizer state at iteration 26 to ./logs/44k/G_1800.pth
|
131 |
+
2023-05-03 13:44:35,263 44k INFO Saving model and optimizer state at iteration 26 to ./logs/44k/D_1800.pth
|
132 |
+
2023-05-03 13:44:35,805 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1600.pth
|
133 |
+
2023-05-03 13:44:35,834 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1600.pth
|
134 |
+
2023-05-03 13:45:07,327 44k INFO ====> Epoch: 26, cost 58.52 s
|
135 |
+
2023-05-03 13:45:43,605 44k INFO Train Epoch: 27 [75%]
|
136 |
+
2023-05-03 13:45:43,606 44k INFO Losses: [2.493234395980835, 2.7097132205963135, 7.805803298950195, 22.947893142700195, 1.7040966749191284], step: 1900, lr: 9.967550730505221e-05, reference_loss: 37.66073989868164
|
137 |
+
2023-05-03 13:45:48,718 44k INFO Saving model and optimizer state at iteration 27 to ./logs/44k/G_1900.pth
|
138 |
+
2023-05-03 13:45:50,379 44k INFO Saving model and optimizer state at iteration 27 to ./logs/44k/D_1900.pth
|
139 |
+
2023-05-03 13:45:50,920 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1700.pth
|
140 |
+
2023-05-03 13:45:50,949 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1700.pth
|
141 |
+
2023-05-03 13:46:01,578 44k INFO ====> Epoch: 27, cost 54.25 s
|
142 |
+
2023-05-03 13:46:48,254 44k INFO ====> Epoch: 28, cost 46.68 s
|
143 |
+
2023-05-03 13:46:59,224 44k INFO Train Epoch: 29 [15%]
|
144 |
+
2023-05-03 13:46:59,225 44k INFO Losses: [2.339369773864746, 2.4442389011383057, 11.337064743041992, 22.017444610595703, 1.5041507482528687], step: 2000, lr: 9.965058998565574e-05, reference_loss: 39.642269134521484
|
145 |
+
2023-05-03 13:47:04,425 44k INFO Saving model and optimizer state at iteration 29 to ./logs/44k/G_2000.pth
|
146 |
+
2023-05-03 13:47:06,082 44k INFO Saving model and optimizer state at iteration 29 to ./logs/44k/D_2000.pth
|
147 |
+
2023-05-03 13:47:06,617 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1800.pth
|
148 |
+
2023-05-03 13:47:06,646 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1800.pth
|
149 |
+
2023-05-03 13:47:42,811 44k INFO ====> Epoch: 29, cost 54.56 s
|
150 |
+
2023-05-03 13:48:11,315 44k INFO Train Epoch: 30 [56%]
|
151 |
+
2023-05-03 13:48:11,316 44k INFO Losses: [2.388498306274414, 2.5379557609558105, 7.413846015930176, 23.07448959350586, 1.256014108657837], step: 2100, lr: 9.963813366190753e-05, reference_loss: 36.67080307006836
|
152 |
+
2023-05-03 13:48:16,439 44k INFO Saving model and optimizer state at iteration 30 to ./logs/44k/G_2100.pth
|
153 |
+
2023-05-03 13:48:18,342 44k INFO Saving model and optimizer state at iteration 30 to ./logs/44k/D_2100.pth
|
154 |
+
2023-05-03 13:48:18,874 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_1900.pth
|
155 |
+
2023-05-03 13:48:18,903 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_1900.pth
|
156 |
+
2023-05-03 13:48:37,697 44k INFO ====> Epoch: 30, cost 54.89 s
|
157 |
+
2023-05-03 13:49:23,424 44k INFO Train Epoch: 31 [97%]
|
158 |
+
2023-05-03 13:49:23,424 44k INFO Losses: [2.6424458026885986, 2.4859418869018555, 9.348760604858398, 21.602563858032227, 1.1841977834701538], step: 2200, lr: 9.962567889519979e-05, reference_loss: 37.26390838623047
|
159 |
+
2023-05-03 13:49:28,489 44k INFO Saving model and optimizer state at iteration 31 to ./logs/44k/G_2200.pth
|
160 |
+
2023-05-03 13:49:30,184 44k INFO Saving model and optimizer state at iteration 31 to ./logs/44k/D_2200.pth
|
161 |
+
2023-05-03 13:49:30,714 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2000.pth
|
162 |
+
2023-05-03 13:49:30,743 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2000.pth
|
163 |
+
2023-05-03 13:49:31,809 44k INFO ====> Epoch: 31, cost 54.11 s
|
164 |
+
2023-05-03 13:50:18,662 44k INFO ====> Epoch: 32, cost 46.85 s
|
165 |
+
2023-05-03 13:50:39,138 44k INFO Train Epoch: 33 [38%]
|
166 |
+
2023-05-03 13:50:39,139 44k INFO Losses: [2.171383857727051, 2.7099664211273193, 9.955101013183594, 30.65108299255371, 1.07896089553833], step: 2300, lr: 9.960077403212722e-05, reference_loss: 46.56649398803711
|
167 |
+
2023-05-03 13:50:44,292 44k INFO Saving model and optimizer state at iteration 33 to ./logs/44k/G_2300.pth
|
168 |
+
2023-05-03 13:50:45,922 44k INFO Saving model and optimizer state at iteration 33 to ./logs/44k/D_2300.pth
|
169 |
+
2023-05-03 13:50:46,454 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2100.pth
|
170 |
+
2023-05-03 13:50:46,483 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2100.pth
|
171 |
+
2023-05-03 13:51:12,651 44k INFO ====> Epoch: 33, cost 53.99 s
|
172 |
+
2023-05-03 13:51:51,078 44k INFO Train Epoch: 34 [79%]
|
173 |
+
2023-05-03 13:51:51,079 44k INFO Losses: [2.7176640033721924, 2.151402473449707, 7.5227837562561035, 22.640493392944336, 1.2028143405914307], step: 2400, lr: 9.95883239353732e-05, reference_loss: 36.23515701293945
|
174 |
+
2023-05-03 13:51:56,385 44k INFO Saving model and optimizer state at iteration 34 to ./logs/44k/G_2400.pth
|
175 |
+
2023-05-03 13:51:58,063 44k INFO Saving model and optimizer state at iteration 34 to ./logs/44k/D_2400.pth
|
176 |
+
2023-05-03 13:51:58,598 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2200.pth
|
177 |
+
2023-05-03 13:51:58,627 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2200.pth
|
178 |
+
2023-05-03 13:52:07,487 44k INFO ====> Epoch: 34, cost 54.84 s
|
179 |
+
2023-05-03 13:52:54,709 44k INFO ====> Epoch: 35, cost 47.22 s
|
180 |
+
2023-05-03 13:53:07,508 44k INFO Train Epoch: 36 [20%]
|
181 |
+
2023-05-03 13:53:07,508 44k INFO Losses: [2.1929237842559814, 2.455925941467285, 8.182902336120605, 22.028627395629883, 1.0674852132797241], step: 2500, lr: 9.956342841045691e-05, reference_loss: 35.92786407470703
|
182 |
+
2023-05-03 13:53:12,777 44k INFO Saving model and optimizer state at iteration 36 to ./logs/44k/G_2500.pth
|
183 |
+
2023-05-03 13:53:14,484 44k INFO Saving model and optimizer state at iteration 36 to ./logs/44k/D_2500.pth
|
184 |
+
2023-05-03 13:53:15,023 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2300.pth
|
185 |
+
2023-05-03 13:53:15,052 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2300.pth
|
186 |
+
2023-05-03 13:53:50,858 44k INFO ====> Epoch: 36, cost 56.15 s
|
187 |
+
2023-05-03 13:54:21,588 44k INFO Train Epoch: 37 [61%]
|
188 |
+
2023-05-03 13:54:21,589 44k INFO Losses: [2.3986458778381348, 2.24863338470459, 10.338911056518555, 22.920942306518555, 1.7659378051757812], step: 2600, lr: 9.95509829819056e-05, reference_loss: 39.673072814941406
|
189 |
+
2023-05-03 13:54:26,751 44k INFO Saving model and optimizer state at iteration 37 to ./logs/44k/G_2600.pth
|
190 |
+
2023-05-03 13:54:28,569 44k INFO Saving model and optimizer state at iteration 37 to ./logs/44k/D_2600.pth
|
191 |
+
2023-05-03 13:54:29,104 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2400.pth
|
192 |
+
2023-05-03 13:54:29,132 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2400.pth
|
193 |
+
2023-05-03 13:54:45,792 44k INFO ====> Epoch: 37, cost 54.93 s
|
194 |
+
2023-05-03 13:55:32,897 44k INFO ====> Epoch: 38, cost 47.11 s
|
195 |
+
2023-05-03 13:55:37,793 44k INFO Train Epoch: 39 [1%]
|
196 |
+
2023-05-03 13:55:37,793 44k INFO Losses: [2.5867271423339844, 2.1795637607574463, 8.613794326782227, 22.480947494506836, 1.3132673501968384], step: 2700, lr: 9.952609679164422e-05, reference_loss: 37.17429733276367
|
197 |
+
2023-05-03 13:55:42,846 44k INFO Saving model and optimizer state at iteration 39 to ./logs/44k/G_2700.pth
|
198 |
+
2023-05-03 13:55:44,488 44k INFO Saving model and optimizer state at iteration 39 to ./logs/44k/D_2700.pth
|
199 |
+
2023-05-03 13:55:45,015 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2500.pth
|
200 |
+
2023-05-03 13:55:45,043 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2500.pth
|
201 |
+
2023-05-03 13:56:26,924 44k INFO ====> Epoch: 39, cost 54.03 s
|
202 |
+
2023-05-03 13:56:49,364 44k INFO Train Epoch: 40 [42%]
|
203 |
+
2023-05-03 13:56:49,364 44k INFO Losses: [2.073730945587158, 2.529214382171631, 10.178397178649902, 23.654401779174805, 1.5418543815612793], step: 2800, lr: 9.951365602954526e-05, reference_loss: 39.97760009765625
|
204 |
+
2023-05-03 13:56:54,566 44k INFO Saving model and optimizer state at iteration 40 to ./logs/44k/G_2800.pth
|
205 |
+
2023-05-03 13:56:56,221 44k INFO Saving model and optimizer state at iteration 40 to ./logs/44k/D_2800.pth
|
206 |
+
2023-05-03 13:56:56,751 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2600.pth
|
207 |
+
2023-05-03 13:56:56,779 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2600.pth
|
208 |
+
2023-05-03 13:57:21,240 44k INFO ====> Epoch: 40, cost 54.32 s
|
209 |
+
2023-05-03 13:58:01,611 44k INFO Train Epoch: 41 [83%]
|
210 |
+
2023-05-03 13:58:01,612 44k INFO Losses: [2.2060742378234863, 2.293834686279297, 9.187530517578125, 23.000995635986328, 1.1135773658752441], step: 2900, lr: 9.950121682254156e-05, reference_loss: 37.8020133972168
|
211 |
+
2023-05-03 13:58:06,827 44k INFO Saving model and optimizer state at iteration 41 to ./logs/44k/G_2900.pth
|
212 |
+
2023-05-03 13:58:08,418 44k INFO Saving model and optimizer state at iteration 41 to ./logs/44k/D_2900.pth
|
213 |
+
2023-05-03 13:58:08,952 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2700.pth
|
214 |
+
2023-05-03 13:58:08,981 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2700.pth
|
215 |
+
2023-05-03 13:58:16,038 44k INFO ====> Epoch: 41, cost 54.80 s
|
216 |
+
2023-05-03 13:59:03,415 44k INFO ====> Epoch: 42, cost 47.38 s
|
217 |
+
2023-05-03 13:59:17,934 44k INFO Train Epoch: 43 [24%]
|
218 |
+
2023-05-03 13:59:17,935 44k INFO Losses: [2.6567223072052, 2.23923921585083, 8.49673080444336, 21.460689544677734, 1.3385343551635742], step: 3000, lr: 9.947634307304244e-05, reference_loss: 36.191917419433594
|
219 |
+
2023-05-03 13:59:23,051 44k INFO Saving model and optimizer state at iteration 43 to ./logs/44k/G_3000.pth
|
220 |
+
2023-05-03 13:59:24,847 44k INFO Saving model and optimizer state at iteration 43 to ./logs/44k/D_3000.pth
|
221 |
+
2023-05-03 13:59:25,379 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2800.pth
|
222 |
+
2023-05-03 13:59:25,407 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2800.pth
|
223 |
+
2023-05-03 13:59:57,447 44k INFO ====> Epoch: 43, cost 54.03 s
|
224 |
+
2023-05-03 14:00:29,515 44k INFO Train Epoch: 44 [65%]
|
225 |
+
2023-05-03 14:00:29,516 44k INFO Losses: [2.4680213928222656, 2.120514154434204, 9.469073295593262, 20.154348373413086, 0.9433535933494568], step: 3100, lr: 9.94639085301583e-05, reference_loss: 35.15530776977539
|
226 |
+
2023-05-03 14:00:34,549 44k INFO Saving model and optimizer state at iteration 44 to ./logs/44k/G_3100.pth
|
227 |
+
2023-05-03 14:00:36,255 44k INFO Saving model and optimizer state at iteration 44 to ./logs/44k/D_3100.pth
|
228 |
+
2023-05-03 14:00:36,789 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_2900.pth
|
229 |
+
2023-05-03 14:00:36,817 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_2900.pth
|
230 |
+
2023-05-03 14:00:51,446 44k INFO ====> Epoch: 44, cost 54.00 s
|
231 |
+
2023-05-03 14:01:38,351 44k INFO ====> Epoch: 45, cost 46.91 s
|
232 |
+
2023-05-03 14:01:45,125 44k INFO Train Epoch: 46 [6%]
|
233 |
+
2023-05-03 14:01:45,126 44k INFO Losses: [2.278339385986328, 2.6492905616760254, 11.130102157592773, 22.5175838470459, 1.442008137702942], step: 3200, lr: 9.943904410714931e-05, reference_loss: 40.01732635498047
|
234 |
+
2023-05-03 14:01:50,356 44k INFO Saving model and optimizer state at iteration 46 to ./logs/44k/G_3200.pth
|
235 |
+
2023-05-03 14:01:51,996 44k INFO Saving model and optimizer state at iteration 46 to ./logs/44k/D_3200.pth
|
236 |
+
2023-05-03 14:01:52,543 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3000.pth
|
237 |
+
2023-05-03 14:01:52,571 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3000.pth
|
238 |
+
2023-05-03 14:02:32,921 44k INFO ====> Epoch: 46, cost 54.57 s
|
239 |
+
2023-05-03 14:02:57,250 44k INFO Train Epoch: 47 [46%]
|
240 |
+
2023-05-03 14:02:57,251 44k INFO Losses: [2.453352212905884, 2.0893375873565674, 9.351663589477539, 20.12795639038086, 1.1927286386489868], step: 3300, lr: 9.942661422663591e-05, reference_loss: 35.21503829956055
|
241 |
+
2023-05-03 14:03:02,367 44k INFO Saving model and optimizer state at iteration 47 to ./logs/44k/G_3300.pth
|
242 |
+
2023-05-03 14:03:04,113 44k INFO Saving model and optimizer state at iteration 47 to ./logs/44k/D_3300.pth
|
243 |
+
2023-05-03 14:03:04,642 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3100.pth
|
244 |
+
2023-05-03 14:03:04,671 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3100.pth
|
245 |
+
2023-05-03 14:03:27,420 44k INFO ====> Epoch: 47, cost 54.50 s
|
246 |
+
2023-05-03 14:04:09,331 44k INFO Train Epoch: 48 [87%]
|
247 |
+
2023-05-03 14:04:09,332 44k INFO Losses: [2.1655383110046387, 2.269059896469116, 12.266657829284668, 22.0493106842041, 0.909777045249939], step: 3400, lr: 9.941418589985758e-05, reference_loss: 39.660343170166016
|
248 |
+
2023-05-03 14:04:14,409 44k INFO Saving model and optimizer state at iteration 48 to ./logs/44k/G_3400.pth
|
249 |
+
2023-05-03 14:04:15,991 44k INFO Saving model and optimizer state at iteration 48 to ./logs/44k/D_3400.pth
|
250 |
+
2023-05-03 14:04:16,523 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3200.pth
|
251 |
+
2023-05-03 14:04:16,552 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3200.pth
|
252 |
+
2023-05-03 14:04:21,824 44k INFO ====> Epoch: 48, cost 54.40 s
|
253 |
+
2023-05-03 14:05:08,501 44k INFO ====> Epoch: 49, cost 46.68 s
|
254 |
+
2023-05-03 14:05:25,017 44k INFO Train Epoch: 50 [28%]
|
255 |
+
2023-05-03 14:05:25,018 44k INFO Losses: [2.5194897651672363, 2.653073310852051, 9.11046028137207, 19.312061309814453, 1.3811150789260864], step: 3500, lr: 9.938933390672926e-05, reference_loss: 34.976200103759766
|
256 |
+
2023-05-03 14:05:30,215 44k INFO Saving model and optimizer state at iteration 50 to ./logs/44k/G_3500.pth
|
257 |
+
2023-05-03 14:05:31,888 44k INFO Saving model and optimizer state at iteration 50 to ./logs/44k/D_3500.pth
|
258 |
+
2023-05-03 14:05:32,422 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3300.pth
|
259 |
+
2023-05-03 14:05:32,451 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3300.pth
|
260 |
+
2023-05-03 14:06:03,174 44k INFO ====> Epoch: 50, cost 54.67 s
|
261 |
+
2023-05-03 14:06:40,848 44k INFO Train Epoch: 51 [69%]
|
262 |
+
2023-05-03 14:06:40,849 44k INFO Losses: [2.2235589027404785, 2.4128994941711426, 9.080496788024902, 21.989484786987305, 1.191401720046997], step: 3600, lr: 9.937691023999092e-05, reference_loss: 36.89784240722656
|
263 |
+
2023-05-03 14:06:46,122 44k INFO Saving model and optimizer state at iteration 51 to ./logs/44k/G_3600.pth
|
264 |
+
2023-05-03 14:06:47,781 44k INFO Saving model and optimizer state at iteration 51 to ./logs/44k/D_3600.pth
|
265 |
+
2023-05-03 14:06:48,316 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3400.pth
|
266 |
+
2023-05-03 14:06:48,345 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3400.pth
|
267 |
+
2023-05-03 14:07:01,367 44k INFO ====> Epoch: 51, cost 58.19 s
|
268 |
+
2023-05-03 14:07:54,030 44k INFO ====> Epoch: 52, cost 52.66 s
|
269 |
+
2023-05-03 14:08:02,552 44k INFO Train Epoch: 53 [10%]
|
270 |
+
2023-05-03 14:08:02,554 44k INFO Losses: [2.477534770965576, 2.28418231010437, 7.059985637664795, 17.771181106567383, 1.0846796035766602], step: 3700, lr: 9.935206756519513e-05, reference_loss: 30.677562713623047
|
271 |
+
2023-05-03 14:08:07,681 44k INFO Saving model and optimizer state at iteration 53 to ./logs/44k/G_3700.pth
|
272 |
+
2023-05-03 14:08:09,439 44k INFO Saving model and optimizer state at iteration 53 to ./logs/44k/D_3700.pth
|
273 |
+
2023-05-03 14:08:09,972 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3500.pth
|
274 |
+
2023-05-03 14:08:10,000 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3500.pth
|
275 |
+
2023-05-03 14:08:48,144 44k INFO ====> Epoch: 53, cost 54.11 s
|
276 |
+
2023-05-03 14:09:14,429 44k INFO Train Epoch: 54 [51%]
|
277 |
+
2023-05-03 14:09:14,430 44k INFO Losses: [2.4032552242279053, 2.1533122062683105, 9.136382102966309, 19.882036209106445, 1.3597607612609863], step: 3800, lr: 9.933964855674948e-05, reference_loss: 34.93474578857422
|
278 |
+
2023-05-03 14:09:19,621 44k INFO Saving model and optimizer state at iteration 54 to ./logs/44k/G_3800.pth
|
279 |
+
2023-05-03 14:09:21,455 44k INFO Saving model and optimizer state at iteration 54 to ./logs/44k/D_3800.pth
|
280 |
+
2023-05-03 14:09:22,025 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3600.pth
|
281 |
+
2023-05-03 14:09:22,054 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3600.pth
|
282 |
+
2023-05-03 14:09:43,013 44k INFO ====> Epoch: 54, cost 54.87 s
|
283 |
+
2023-05-03 14:10:26,509 44k INFO Train Epoch: 55 [92%]
|
284 |
+
2023-05-03 14:10:26,510 44k INFO Losses: [2.3103950023651123, 2.568009614944458, 9.782731056213379, 21.263418197631836, 1.306481122970581], step: 3900, lr: 9.932723110067987e-05, reference_loss: 37.23103332519531
|
285 |
+
2023-05-03 14:10:31,549 44k INFO Saving model and optimizer state at iteration 55 to ./logs/44k/G_3900.pth
|
286 |
+
2023-05-03 14:10:33,204 44k INFO Saving model and optimizer state at iteration 55 to ./logs/44k/D_3900.pth
|
287 |
+
2023-05-03 14:10:33,736 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3700.pth
|
288 |
+
2023-05-03 14:10:33,764 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3700.pth
|
289 |
+
2023-05-03 14:10:37,205 44k INFO ====> Epoch: 55, cost 54.19 s
|
290 |
+
2023-05-03 14:11:24,314 44k INFO ====> Epoch: 56, cost 47.11 s
|
291 |
+
2023-05-03 14:11:42,580 44k INFO Train Epoch: 57 [32%]
|
292 |
+
2023-05-03 14:11:42,581 44k INFO Losses: [2.491316080093384, 2.3412065505981445, 10.123672485351562, 21.33326530456543, 0.7533155083656311], step: 4000, lr: 9.930240084489267e-05, reference_loss: 37.04277420043945
|
293 |
+
2023-05-03 14:11:47,778 44k INFO Saving model and optimizer state at iteration 57 to ./logs/44k/G_4000.pth
|
294 |
+
2023-05-03 14:11:49,466 44k INFO Saving model and optimizer state at iteration 57 to ./logs/44k/D_4000.pth
|
295 |
+
2023-05-03 14:11:49,997 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3800.pth
|
296 |
+
2023-05-03 14:11:50,026 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3800.pth
|
297 |
+
2023-05-03 14:12:18,659 44k INFO ====> Epoch: 57, cost 54.34 s
|
298 |
+
2023-05-03 14:12:54,591 44k INFO Train Epoch: 58 [73%]
|
299 |
+
2023-05-03 14:12:54,592 44k INFO Losses: [2.2970752716064453, 2.240769386291504, 9.480262756347656, 21.239805221557617, 1.4316742420196533], step: 4100, lr: 9.928998804478705e-05, reference_loss: 36.6895866394043
|
300 |
+
2023-05-03 14:12:59,699 44k INFO Saving model and optimizer state at iteration 58 to ./logs/44k/G_4100.pth
|
301 |
+
2023-05-03 14:13:01,526 44k INFO Saving model and optimizer state at iteration 58 to ./logs/44k/D_4100.pth
|
302 |
+
2023-05-03 14:13:02,062 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_3900.pth
|
303 |
+
2023-05-03 14:13:02,090 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_3900.pth
|
304 |
+
2023-05-03 14:13:13,278 44k INFO ====> Epoch: 58, cost 54.62 s
|
305 |
+
2023-05-03 14:13:59,954 44k INFO ====> Epoch: 59, cost 46.68 s
|
306 |
+
2023-05-03 14:14:10,277 44k INFO Train Epoch: 60 [14%]
|
307 |
+
2023-05-03 14:14:10,278 44k INFO Losses: [2.385362148284912, 2.30918550491333, 8.672965049743652, 19.744047164916992, 1.3143541812896729], step: 4200, lr: 9.926516709918191e-05, reference_loss: 34.4259147644043
|
308 |
+
2023-05-03 14:14:15,357 44k INFO Saving model and optimizer state at iteration 60 to ./logs/44k/G_4200.pth
|
309 |
+
2023-05-03 14:14:17,015 44k INFO Saving model and optimizer state at iteration 60 to ./logs/44k/D_4200.pth
|
310 |
+
2023-05-03 14:14:17,544 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4000.pth
|
311 |
+
2023-05-03 14:14:17,573 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4000.pth
|
312 |
+
2023-05-03 14:14:53,940 44k INFO ====> Epoch: 60, cost 53.99 s
|
313 |
+
2023-05-03 14:15:21,698 44k INFO Train Epoch: 61 [55%]
|
314 |
+
2023-05-03 14:15:21,699 44k INFO Losses: [2.2300548553466797, 2.3148417472839355, 10.437084197998047, 22.084848403930664, 1.2689226865768433], step: 4300, lr: 9.92527589532945e-05, reference_loss: 38.335750579833984
|
315 |
+
2023-05-03 14:15:26,958 44k INFO Saving model and optimizer state at iteration 61 to ./logs/44k/G_4300.pth
|
316 |
+
2023-05-03 14:15:28,558 44k INFO Saving model and optimizer state at iteration 61 to ./logs/44k/D_4300.pth
|
317 |
+
2023-05-03 14:15:29,093 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4100.pth
|
318 |
+
2023-05-03 14:15:29,121 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4100.pth
|
319 |
+
2023-05-03 14:15:48,194 44k INFO ====> Epoch: 61, cost 54.25 s
|
320 |
+
2023-05-03 14:16:33,630 44k INFO Train Epoch: 62 [96%]
|
321 |
+
2023-05-03 14:16:33,630 44k INFO Losses: [2.3527278900146484, 2.323845863342285, 9.502860069274902, 22.04048728942871, 0.9736533761024475], step: 4400, lr: 9.924035235842533e-05, reference_loss: 37.193572998046875
|
322 |
+
2023-05-03 14:16:38,801 44k INFO Saving model and optimizer state at iteration 62 to ./logs/44k/G_4400.pth
|
323 |
+
2023-05-03 14:16:40,458 44k INFO Saving model and optimizer state at iteration 62 to ./logs/44k/D_4400.pth
|
324 |
+
2023-05-03 14:16:40,992 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4200.pth
|
325 |
+
2023-05-03 14:16:41,021 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4200.pth
|
326 |
+
2023-05-03 14:16:42,687 44k INFO ====> Epoch: 62, cost 54.49 s
|
327 |
+
2023-05-03 14:17:34,840 44k INFO ====> Epoch: 63, cost 52.15 s
|
328 |
+
2023-05-03 14:17:54,933 44k INFO Train Epoch: 64 [37%]
|
329 |
+
2023-05-03 14:17:54,934 44k INFO Losses: [2.385552406311035, 2.1727263927459717, 9.087684631347656, 21.040884017944336, 1.0232913494110107], step: 4500, lr: 9.921554382096622e-05, reference_loss: 35.710140228271484
|
330 |
+
2023-05-03 14:18:00,018 44k INFO Saving model and optimizer state at iteration 64 to ./logs/44k/G_4500.pth
|
331 |
+
2023-05-03 14:18:01,734 44k INFO Saving model and optimizer state at iteration 64 to ./logs/44k/D_4500.pth
|
332 |
+
2023-05-03 14:18:02,268 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4300.pth
|
333 |
+
2023-05-03 14:18:02,296 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4300.pth
|
334 |
+
2023-05-03 14:18:29,408 44k INFO ====> Epoch: 64, cost 54.57 s
|
335 |
+
2023-05-03 14:19:07,204 44k INFO Train Epoch: 65 [77%]
|
336 |
+
2023-05-03 14:19:07,205 44k INFO Losses: [2.2484593391418457, 2.5644028186798096, 11.485355377197266, 21.17015838623047, 1.1813242435455322], step: 4600, lr: 9.92031418779886e-05, reference_loss: 38.64970016479492
|
337 |
+
2023-05-03 14:19:12,347 44k INFO Saving model and optimizer state at iteration 65 to ./logs/44k/G_4600.pth
|
338 |
+
2023-05-03 14:19:14,047 44k INFO Saving model and optimizer state at iteration 65 to ./logs/44k/D_4600.pth
|
339 |
+
2023-05-03 14:19:14,583 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4400.pth
|
340 |
+
2023-05-03 14:19:14,611 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4400.pth
|
341 |
+
2023-05-03 14:19:24,026 44k INFO ====> Epoch: 65, cost 54.62 s
|
342 |
+
2023-05-03 14:20:11,109 44k INFO ====> Epoch: 66, cost 47.08 s
|
343 |
+
2023-05-03 14:20:23,165 44k INFO Train Epoch: 67 [18%]
|
344 |
+
2023-05-03 14:20:23,166 44k INFO Losses: [2.3424651622772217, 2.569873332977295, 7.83887243270874, 22.892330169677734, 1.65440034866333], step: 4700, lr: 9.917834264256819e-05, reference_loss: 37.29793930053711
|
345 |
+
2023-05-03 14:20:28,353 44k INFO Saving model and optimizer state at iteration 67 to ./logs/44k/G_4700.pth
|
346 |
+
2023-05-03 14:20:30,072 44k INFO Saving model and optimizer state at iteration 67 to ./logs/44k/D_4700.pth
|
347 |
+
2023-05-03 14:20:30,606 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4500.pth
|
348 |
+
2023-05-03 14:20:30,635 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4500.pth
|
349 |
+
2023-05-03 14:21:05,218 44k INFO ====> Epoch: 67, cost 54.11 s
|
350 |
+
2023-05-03 14:21:35,022 44k INFO Train Epoch: 68 [59%]
|
351 |
+
2023-05-03 14:21:35,023 44k INFO Losses: [2.3991572856903076, 2.080165386199951, 10.898751258850098, 20.9050235748291, 0.7145375609397888], step: 4800, lr: 9.916594534973787e-05, reference_loss: 36.99763488769531
|
352 |
+
2023-05-03 14:21:40,023 44k INFO Saving model and optimizer state at iteration 68 to ./logs/44k/G_4800.pth
|
353 |
+
2023-05-03 14:21:41,821 44k INFO Saving model and optimizer state at iteration 68 to ./logs/44k/D_4800.pth
|
354 |
+
2023-05-03 14:21:42,354 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4600.pth
|
355 |
+
2023-05-03 14:21:42,383 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4600.pth
|
356 |
+
2023-05-03 14:21:59,598 44k INFO ====> Epoch: 68, cost 54.38 s
|
357 |
+
2023-05-03 14:22:46,885 44k INFO ====> Epoch: 69, cost 47.29 s
|
358 |
+
2023-05-03 14:22:51,107 44k INFO Train Epoch: 70 [0%]
|
359 |
+
2023-05-03 14:22:51,108 44k INFO Losses: [2.243220329284668, 2.3275976181030273, 9.493850708007812, 20.528539657592773, 1.0689058303833008], step: 4900, lr: 9.914115541286833e-05, reference_loss: 35.662113189697266
|
360 |
+
2023-05-03 14:22:56,175 44k INFO Saving model and optimizer state at iteration 70 to ./logs/44k/G_4900.pth
|
361 |
+
2023-05-03 14:22:57,868 44k INFO Saving model and optimizer state at iteration 70 to ./logs/44k/D_4900.pth
|
362 |
+
2023-05-03 14:22:58,400 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4700.pth
|
363 |
+
2023-05-03 14:22:58,429 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4700.pth
|
364 |
+
2023-05-03 14:23:41,035 44k INFO ====> Epoch: 70, cost 54.15 s
|
365 |
+
2023-05-03 14:24:04,991 44k INFO Train Epoch: 71 [41%]
|
366 |
+
2023-05-03 14:24:04,992 44k INFO Losses: [2.3157591819763184, 2.332456588745117, 7.5815205574035645, 22.40475845336914, 1.2106596231460571], step: 5000, lr: 9.912876276844171e-05, reference_loss: 35.84515380859375
|
367 |
+
2023-05-03 14:24:10,111 44k INFO Saving model and optimizer state at iteration 71 to ./logs/44k/G_5000.pth
|
368 |
+
2023-05-03 14:24:11,823 44k INFO Saving model and optimizer state at iteration 71 to ./logs/44k/D_5000.pth
|
369 |
+
2023-05-03 14:24:12,362 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4800.pth
|
370 |
+
2023-05-03 14:24:12,391 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4800.pth
|
371 |
+
2023-05-03 14:24:40,346 44k INFO ====> Epoch: 71, cost 59.31 s
|
372 |
+
2023-05-03 14:25:19,763 44k INFO Train Epoch: 72 [82%]
|
373 |
+
2023-05-03 14:25:19,763 44k INFO Losses: [2.090325117111206, 2.388118267059326, 12.023096084594727, 22.705028533935547, 1.1269657611846924], step: 5100, lr: 9.911637167309565e-05, reference_loss: 40.33353042602539
|
374 |
+
2023-05-03 14:25:25,025 44k INFO Saving model and optimizer state at iteration 72 to ./logs/44k/G_5100.pth
|
375 |
+
2023-05-03 14:25:26,635 44k INFO Saving model and optimizer state at iteration 72 to ./logs/44k/D_5100.pth
|
376 |
+
2023-05-03 14:25:27,169 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_4900.pth
|
377 |
+
2023-05-03 14:25:27,198 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_4900.pth
|
378 |
+
2023-05-03 14:25:34,804 44k INFO ====> Epoch: 72, cost 54.46 s
|
379 |
+
2023-05-03 14:26:21,926 44k INFO ====> Epoch: 73, cost 47.12 s
|
380 |
+
2023-05-03 14:26:35,964 44k INFO Train Epoch: 74 [23%]
|
381 |
+
2023-05-03 14:26:35,965 44k INFO Losses: [2.2803421020507812, 2.3115463256835938, 10.413973808288574, 23.707111358642578, 1.475788950920105], step: 5200, lr: 9.909159412887068e-05, reference_loss: 40.18876266479492
|
382 |
+
2023-05-03 14:26:41,101 44k INFO Saving model and optimizer state at iteration 74 to ./logs/44k/G_5200.pth
|
383 |
+
2023-05-03 14:26:42,833 44k INFO Saving model and optimizer state at iteration 74 to ./logs/44k/D_5200.pth
|
384 |
+
2023-05-03 14:26:43,370 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5000.pth
|
385 |
+
2023-05-03 14:26:43,398 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5000.pth
|
386 |
+
2023-05-03 14:27:16,107 44k INFO ====> Epoch: 74, cost 54.18 s
|
387 |
+
2023-05-03 14:27:47,480 44k INFO Train Epoch: 75 [63%]
|
388 |
+
2023-05-03 14:27:47,481 44k INFO Losses: [2.234922409057617, 2.467733860015869, 9.63402271270752, 20.404632568359375, 1.2089588642120361], step: 5300, lr: 9.907920767960457e-05, reference_loss: 35.95026779174805
|
389 |
+
2023-05-03 14:27:52,458 44k INFO Saving model and optimizer state at iteration 75 to ./logs/44k/G_5300.pth
|
390 |
+
2023-05-03 14:27:54,283 44k INFO Saving model and optimizer state at iteration 75 to ./logs/44k/D_5300.pth
|
391 |
+
2023-05-03 14:27:54,817 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5100.pth
|
392 |
+
2023-05-03 14:27:54,845 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5100.pth
|
393 |
+
2023-05-03 14:28:10,119 44k INFO ====> Epoch: 75, cost 54.01 s
|
394 |
+
2023-05-03 14:28:56,847 44k INFO ====> Epoch: 76, cost 46.73 s
|
395 |
+
2023-05-03 14:29:02,893 44k INFO Train Epoch: 77 [4%]
|
396 |
+
2023-05-03 14:29:02,893 44k INFO Losses: [2.6155261993408203, 2.135758876800537, 5.038893222808838, 17.311443328857422, 1.409765601158142], step: 5400, lr: 9.905443942579728e-05, reference_loss: 28.51138687133789
|
397 |
+
2023-05-03 14:29:07,925 44k INFO Saving model and optimizer state at iteration 77 to ./logs/44k/G_5400.pth
|
398 |
+
2023-05-03 14:29:09,665 44k INFO Saving model and optimizer state at iteration 77 to ./logs/44k/D_5400.pth
|
399 |
+
2023-05-03 14:29:10,192 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5200.pth
|
400 |
+
2023-05-03 14:29:10,221 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5200.pth
|
401 |
+
2023-05-03 14:29:51,047 44k INFO ====> Epoch: 77, cost 54.20 s
|
402 |
+
2023-05-03 14:30:14,772 44k INFO Train Epoch: 78 [45%]
|
403 |
+
2023-05-03 14:30:14,772 44k INFO Losses: [2.413071393966675, 2.4141740798950195, 6.657437801361084, 20.914470672607422, 1.1908607482910156], step: 5500, lr: 9.904205762086905e-05, reference_loss: 33.59001541137695
|
404 |
+
2023-05-03 14:30:20,010 44k INFO Saving model and optimizer state at iteration 78 to ./logs/44k/G_5500.pth
|
405 |
+
2023-05-03 14:30:21,621 44k INFO Saving model and optimizer state at iteration 78 to ./logs/44k/D_5500.pth
|
406 |
+
2023-05-03 14:30:22,193 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5300.pth
|
407 |
+
2023-05-03 14:30:22,222 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5300.pth
|
408 |
+
2023-05-03 14:30:45,550 44k INFO ====> Epoch: 78, cost 54.50 s
|
409 |
+
2023-05-03 14:31:29,477 44k INFO Train Epoch: 79 [86%]
|
410 |
+
2023-05-03 14:31:29,478 44k INFO Losses: [2.4432146549224854, 2.3251218795776367, 12.018153190612793, 20.31390380859375, 1.342065453529358], step: 5600, lr: 9.902967736366644e-05, reference_loss: 38.44245910644531
|
411 |
+
2023-05-03 14:31:34,567 44k INFO Saving model and optimizer state at iteration 79 to ./logs/44k/G_5600.pth
|
412 |
+
2023-05-03 14:31:36,333 44k INFO Saving model and optimizer state at iteration 79 to ./logs/44k/D_5600.pth
|
413 |
+
2023-05-03 14:31:36,901 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5400.pth
|
414 |
+
2023-05-03 14:31:36,929 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5400.pth
|
415 |
+
2023-05-03 14:31:42,708 44k INFO ====> Epoch: 79, cost 57.16 s
|
416 |
+
2023-05-03 14:32:34,654 44k INFO ====> Epoch: 80, cost 51.95 s
|
417 |
+
2023-05-03 14:32:50,414 44k INFO Train Epoch: 81 [27%]
|
418 |
+
2023-05-03 14:32:50,415 44k INFO Losses: [2.7789437770843506, 2.3213107585906982, 9.386683464050293, 21.96425437927246, 1.3818237781524658], step: 5700, lr: 9.900492149166423e-05, reference_loss: 37.8330192565918
|
419 |
+
2023-05-03 14:32:55,474 44k INFO Saving model and optimizer state at iteration 81 to ./logs/44k/G_5700.pth
|
420 |
+
2023-05-03 14:32:57,143 44k INFO Saving model and optimizer state at iteration 81 to ./logs/44k/D_5700.pth
|
421 |
+
2023-05-03 14:32:57,673 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5500.pth
|
422 |
+
2023-05-03 14:32:57,702 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5500.pth
|
423 |
+
2023-05-03 14:33:28,696 44k INFO ====> Epoch: 81, cost 54.04 s
|
424 |
+
2023-05-03 14:34:01,815 44k INFO Train Epoch: 82 [68%]
|
425 |
+
2023-05-03 14:34:01,816 44k INFO Losses: [2.211815357208252, 2.320291042327881, 5.064887046813965, 23.727109909057617, 0.9825482964515686], step: 5800, lr: 9.899254587647776e-05, reference_loss: 34.3066520690918
|
426 |
+
2023-05-03 14:34:06,827 44k INFO Saving model and optimizer state at iteration 82 to ./logs/44k/G_5800.pth
|
427 |
+
2023-05-03 14:34:08,521 44k INFO Saving model and optimizer state at iteration 82 to ./logs/44k/D_5800.pth
|
428 |
+
2023-05-03 14:34:09,055 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5600.pth
|
429 |
+
2023-05-03 14:34:09,104 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5600.pth
|
430 |
+
2023-05-03 14:34:22,816 44k INFO ====> Epoch: 82, cost 54.12 s
|
431 |
+
2023-05-03 14:35:10,041 44k INFO ====> Epoch: 83, cost 47.23 s
|
432 |
+
2023-05-03 14:35:17,956 44k INFO Train Epoch: 84 [8%]
|
433 |
+
2023-05-03 14:35:17,957 44k INFO Losses: [2.484198570251465, 2.2484796047210693, 10.755833625793457, 20.941932678222656, 1.2315059900283813], step: 5900, lr: 9.896779928676716e-05, reference_loss: 37.661949157714844
|
434 |
+
2023-05-03 14:35:23,191 44k INFO Saving model and optimizer state at iteration 84 to ./logs/44k/G_5900.pth
|
435 |
+
2023-05-03 14:35:24,926 44k INFO Saving model and optimizer state at iteration 84 to ./logs/44k/D_5900.pth
|
436 |
+
2023-05-03 14:35:25,506 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5700.pth
|
437 |
+
2023-05-03 14:35:25,534 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5700.pth
|
438 |
+
2023-05-03 14:36:04,956 44k INFO ====> Epoch: 84, cost 54.91 s
|
439 |
+
2023-05-03 14:36:30,506 44k INFO Train Epoch: 85 [49%]
|
440 |
+
2023-05-03 14:36:30,507 44k INFO Losses: [2.388766288757324, 2.1078274250030518, 8.077157020568848, 20.986013412475586, 1.242616891860962], step: 6000, lr: 9.895542831185631e-05, reference_loss: 34.80238342285156
|
441 |
+
2023-05-03 14:36:35,688 44k INFO Saving model and optimizer state at iteration 85 to ./logs/44k/G_6000.pth
|
442 |
+
2023-05-03 14:36:37,309 44k INFO Saving model and optimizer state at iteration 85 to ./logs/44k/D_6000.pth
|
443 |
+
2023-05-03 14:36:37,844 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5800.pth
|
444 |
+
2023-05-03 14:36:37,875 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5800.pth
|
445 |
+
2023-05-03 14:36:59,366 44k INFO ====> Epoch: 85, cost 54.41 s
|
446 |
+
2023-05-03 14:37:42,269 44k INFO Train Epoch: 86 [90%]
|
447 |
+
2023-05-03 14:37:42,270 44k INFO Losses: [2.4167308807373047, 2.440798044204712, 8.85080623626709, 20.857934951782227, 1.072945237159729], step: 6100, lr: 9.894305888331732e-05, reference_loss: 35.63921356201172
|
448 |
+
2023-05-03 14:37:47,385 44k INFO Saving model and optimizer state at iteration 86 to ./logs/44k/G_6100.pth
|
449 |
+
2023-05-03 14:37:49,228 44k INFO Saving model and optimizer state at iteration 86 to ./logs/44k/D_6100.pth
|
450 |
+
2023-05-03 14:37:49,764 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_5900.pth
|
451 |
+
2023-05-03 14:37:49,810 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_5900.pth
|
452 |
+
2023-05-03 14:37:53,883 44k INFO ====> Epoch: 86, cost 54.52 s
|
453 |
+
2023-05-03 14:38:40,995 44k INFO ====> Epoch: 87, cost 47.11 s
|
454 |
+
2023-05-03 14:38:58,697 44k INFO Train Epoch: 88 [31%]
|
455 |
+
2023-05-03 14:38:58,698 44k INFO Losses: [2.289813995361328, 2.1840624809265137, 11.359431266784668, 23.120685577392578, 1.0711365938186646], step: 6200, lr: 9.891832466458178e-05, reference_loss: 40.02513122558594
|
456 |
+
2023-05-03 14:39:03,798 44k INFO Saving model and optimizer state at iteration 88 to ./logs/44k/G_6200.pth
|
457 |
+
2023-05-03 14:39:05,398 44k INFO Saving model and optimizer state at iteration 88 to ./logs/44k/D_6200.pth
|
458 |
+
2023-05-03 14:39:05,961 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6000.pth
|
459 |
+
2023-05-03 14:39:05,992 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6000.pth
|
460 |
+
2023-05-03 14:39:35,106 44k INFO ====> Epoch: 88, cost 54.11 s
|
461 |
+
2023-05-03 14:40:11,101 44k INFO Train Epoch: 89 [72%]
|
462 |
+
2023-05-03 14:40:11,102 44k INFO Losses: [2.3503124713897705, 2.366110324859619, 7.645803451538086, 23.64559555053711, 1.1122896671295166], step: 6300, lr: 9.89059598739987e-05, reference_loss: 37.12010955810547
|
463 |
+
2023-05-03 14:40:16,443 44k INFO Saving model and optimizer state at iteration 89 to ./logs/44k/G_6300.pth
|
464 |
+
2023-05-03 14:40:18,088 44k INFO Saving model and optimizer state at iteration 89 to ./logs/44k/D_6300.pth
|
465 |
+
2023-05-03 14:40:18,627 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6100.pth
|
466 |
+
2023-05-03 14:40:18,657 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6100.pth
|
467 |
+
2023-05-03 14:40:30,437 44k INFO ====> Epoch: 89, cost 55.33 s
|
468 |
+
2023-05-03 14:41:17,208 44k INFO ====> Epoch: 90, cost 46.77 s
|
469 |
+
2023-05-03 14:41:26,818 44k INFO Train Epoch: 91 [13%]
|
470 |
+
2023-05-03 14:41:26,818 44k INFO Losses: [2.2005672454833984, 2.5051355361938477, 9.279576301574707, 22.79197120666504, 1.0933576822280884], step: 6400, lr: 9.888123492943583e-05, reference_loss: 37.870609283447266
|
471 |
+
2023-05-03 14:41:32,032 44k INFO Saving model and optimizer state at iteration 91 to ./logs/44k/G_6400.pth
|
472 |
+
2023-05-03 14:41:33,728 44k INFO Saving model and optimizer state at iteration 91 to ./logs/44k/D_6400.pth
|
473 |
+
2023-05-03 14:41:34,259 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6200.pth
|
474 |
+
2023-05-03 14:41:34,307 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6200.pth
|
475 |
+
2023-05-03 14:42:11,194 44k INFO ====> Epoch: 91, cost 53.99 s
|
476 |
+
2023-05-03 14:42:38,796 44k INFO Train Epoch: 92 [54%]
|
477 |
+
2023-05-03 14:42:38,796 44k INFO Losses: [2.4646763801574707, 2.076392412185669, 7.404085159301758, 21.058698654174805, 1.0544928312301636], step: 6500, lr: 9.886887477506964e-05, reference_loss: 34.058345794677734
|
478 |
+
2023-05-03 14:42:43,951 44k INFO Saving model and optimizer state at iteration 92 to ./logs/44k/G_6500.pth
|
479 |
+
2023-05-03 14:42:45,633 44k INFO Saving model and optimizer state at iteration 92 to ./logs/44k/D_6500.pth
|
480 |
+
2023-05-03 14:42:46,317 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6300.pth
|
481 |
+
2023-05-03 14:42:46,348 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6300.pth
|
482 |
+
2023-05-03 14:43:06,039 44k INFO ====> Epoch: 92, cost 54.85 s
|
483 |
+
2023-05-03 14:43:55,537 44k INFO Train Epoch: 93 [94%]
|
484 |
+
2023-05-03 14:43:55,537 44k INFO Losses: [2.5010299682617188, 2.3011274337768555, 8.138545036315918, 23.67609405517578, 0.9986937642097473], step: 6600, lr: 9.885651616572276e-05, reference_loss: 37.61549377441406
|
485 |
+
2023-05-03 14:44:00,682 44k INFO Saving model and optimizer state at iteration 93 to ./logs/44k/G_6600.pth
|
486 |
+
2023-05-03 14:44:02,367 44k INFO Saving model and optimizer state at iteration 93 to ./logs/44k/D_6600.pth
|
487 |
+
2023-05-03 14:44:02,916 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6400.pth
|
488 |
+
2023-05-03 14:44:02,945 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6400.pth
|
489 |
+
2023-05-03 14:44:05,398 44k INFO ====> Epoch: 93, cost 59.36 s
|
490 |
+
2023-05-03 14:44:52,822 44k INFO ====> Epoch: 94, cost 47.42 s
|
491 |
+
2023-05-03 14:45:12,476 44k INFO Train Epoch: 95 [35%]
|
492 |
+
2023-05-03 14:45:12,477 44k INFO Losses: [2.310382127761841, 2.2262511253356934, 8.078502655029297, 19.226985931396484, 1.524962306022644], step: 6700, lr: 9.883180358131438e-05, reference_loss: 33.36708450317383
|
493 |
+
2023-05-03 14:45:17,573 44k INFO Saving model and optimizer state at iteration 95 to ./logs/44k/G_6700.pth
|
494 |
+
2023-05-03 14:45:19,270 44k INFO Saving model and optimizer state at iteration 95 to ./logs/44k/D_6700.pth
|
495 |
+
2023-05-03 14:45:19,801 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6500.pth
|
496 |
+
2023-05-03 14:45:19,850 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6500.pth
|
497 |
+
2023-05-03 14:45:47,682 44k INFO ====> Epoch: 95, cost 54.86 s
|
498 |
+
2023-05-03 14:46:25,727 44k INFO Train Epoch: 96 [76%]
|
499 |
+
2023-05-03 14:46:25,728 44k INFO Losses: [2.6544623374938965, 2.2764406204223633, 6.531411170959473, 19.763704299926758, 1.2900036573410034], step: 6800, lr: 9.881944960586671e-05, reference_loss: 32.516021728515625
|
500 |
+
2023-05-03 14:46:31,277 44k INFO Saving model and optimizer state at iteration 96 to ./logs/44k/G_6800.pth
|
501 |
+
2023-05-03 14:46:33,069 44k INFO Saving model and optimizer state at iteration 96 to ./logs/44k/D_6800.pth
|
502 |
+
2023-05-03 14:46:33,648 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6600.pth
|
503 |
+
2023-05-03 14:46:33,677 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6600.pth
|
504 |
+
2023-05-03 14:46:43,773 44k INFO ====> Epoch: 96, cost 56.09 s
|
505 |
+
2023-05-03 14:47:30,320 44k INFO ====> Epoch: 97, cost 46.55 s
|
506 |
+
2023-05-03 14:47:41,778 44k INFO Train Epoch: 98 [17%]
|
507 |
+
2023-05-03 14:47:41,779 44k INFO Losses: [2.525698184967041, 2.3287713527679443, 8.644176483154297, 19.613706588745117, 1.2030137777328491], step: 6900, lr: 9.879474628751914e-05, reference_loss: 34.315364837646484
|
508 |
+
2023-05-03 14:47:47,031 44k INFO Saving model and optimizer state at iteration 98 to ./logs/44k/G_6900.pth
|
509 |
+
2023-05-03 14:47:48,723 44k INFO Saving model and optimizer state at iteration 98 to ./logs/44k/D_6900.pth
|
510 |
+
2023-05-03 14:47:49,257 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6700.pth
|
511 |
+
2023-05-03 14:47:49,286 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6700.pth
|
512 |
+
2023-05-03 14:48:24,690 44k INFO ====> Epoch: 98, cost 54.37 s
|
513 |
+
2023-05-03 14:48:53,995 44k INFO Train Epoch: 99 [58%]
|
514 |
+
2023-05-03 14:48:53,996 44k INFO Losses: [2.533390522003174, 2.219142436981201, 7.681521892547607, 18.070770263671875, 1.1386370658874512], step: 7000, lr: 9.87823969442332e-05, reference_loss: 31.643463134765625
|
515 |
+
2023-05-03 14:48:59,102 44k INFO Saving model and optimizer state at iteration 99 to ./logs/44k/G_7000.pth
|
516 |
+
2023-05-03 14:49:00,957 44k INFO Saving model and optimizer state at iteration 99 to ./logs/44k/D_7000.pth
|
517 |
+
2023-05-03 14:49:01,520 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6800.pth
|
518 |
+
2023-05-03 14:49:01,574 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6800.pth
|
519 |
+
2023-05-03 14:49:19,402 44k INFO ====> Epoch: 99, cost 54.71 s
|
520 |
+
2023-05-03 14:50:12,441 44k INFO Train Epoch: 100 [99%]
|
521 |
+
2023-05-03 14:50:12,442 44k INFO Losses: [2.4274239540100098, 2.672248601913452, 8.535931587219238, 19.511117935180664, 1.0394376516342163], step: 7100, lr: 9.877004914461517e-05, reference_loss: 34.1861572265625
|
522 |
+
2023-05-03 14:50:17,757 44k INFO Saving model and optimizer state at iteration 100 to ./logs/44k/G_7100.pth
|
523 |
+
2023-05-03 14:50:19,557 44k INFO Saving model and optimizer state at iteration 100 to ./logs/44k/D_7100.pth
|
524 |
+
2023-05-03 14:50:20,132 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_6900.pth
|
525 |
+
2023-05-03 14:50:20,163 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_6900.pth
|
526 |
+
2023-05-03 14:50:20,732 44k INFO ====> Epoch: 100, cost 61.33 s
|
527 |
+
2023-05-03 14:51:12,740 44k INFO ====> Epoch: 101, cost 52.01 s
|
528 |
+
2023-05-03 14:51:33,881 44k INFO Train Epoch: 102 [39%]
|
529 |
+
2023-05-03 14:51:33,882 44k INFO Losses: [2.2989039421081543, 2.5206449031829834, 8.606928825378418, 21.207876205444336, 1.165399193763733], step: 7200, lr: 9.874535817561101e-05, reference_loss: 35.79975128173828
|
530 |
+
2023-05-03 14:51:39,061 44k INFO Saving model and optimizer state at iteration 102 to ./logs/44k/G_7200.pth
|
531 |
+
2023-05-03 14:51:40,726 44k INFO Saving model and optimizer state at iteration 102 to ./logs/44k/D_7200.pth
|
532 |
+
2023-05-03 14:51:41,279 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7000.pth
|
533 |
+
2023-05-03 14:51:41,309 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7000.pth
|
534 |
+
2023-05-03 14:52:06,869 44k INFO ====> Epoch: 102, cost 54.13 s
|
535 |
+
2023-05-03 14:52:45,514 44k INFO Train Epoch: 103 [80%]
|
536 |
+
2023-05-03 14:52:45,515 44k INFO Losses: [2.3017122745513916, 2.422215223312378, 9.920147895812988, 18.564119338989258, 0.645395040512085], step: 7300, lr: 9.873301500583906e-05, reference_loss: 33.85358810424805
|
537 |
+
2023-05-03 14:52:50,524 44k INFO Saving model and optimizer state at iteration 103 to ./logs/44k/G_7300.pth
|
538 |
+
2023-05-03 14:52:52,344 44k INFO Saving model and optimizer state at iteration 103 to ./logs/44k/D_7300.pth
|
539 |
+
2023-05-03 14:52:52,900 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7100.pth
|
540 |
+
2023-05-03 14:52:52,953 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7100.pth
|
541 |
+
2023-05-03 14:53:01,109 44k INFO ====> Epoch: 103, cost 54.24 s
|
542 |
+
2023-05-03 14:53:47,732 44k INFO ====> Epoch: 104, cost 46.62 s
|
543 |
+
2023-05-03 14:54:01,060 44k INFO Train Epoch: 105 [21%]
|
544 |
+
2023-05-03 14:54:01,060 44k INFO Losses: [2.75062894821167, 2.412715196609497, 10.205183029174805, 20.36348533630371, 1.4236520528793335], step: 7400, lr: 9.870833329479095e-05, reference_loss: 37.15566635131836
|
545 |
+
2023-05-03 14:54:06,286 44k INFO Saving model and optimizer state at iteration 105 to ./logs/44k/G_7400.pth
|
546 |
+
2023-05-03 14:54:07,938 44k INFO Saving model and optimizer state at iteration 105 to ./logs/44k/D_7400.pth
|
547 |
+
2023-05-03 14:54:08,493 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7200.pth
|
548 |
+
2023-05-03 14:54:08,534 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7200.pth
|
549 |
+
2023-05-03 14:54:41,911 44k INFO ====> Epoch: 105, cost 54.18 s
|
550 |
+
2023-05-03 14:55:13,018 44k INFO Train Epoch: 106 [62%]
|
551 |
+
2023-05-03 14:55:13,019 44k INFO Losses: [2.3272743225097656, 2.4755334854125977, 9.030518531799316, 21.309324264526367, 0.9096031188964844], step: 7500, lr: 9.86959947531291e-05, reference_loss: 36.05225372314453
|
552 |
+
2023-05-03 14:55:18,285 44k INFO Saving model and optimizer state at iteration 106 to ./logs/44k/G_7500.pth
|
553 |
+
2023-05-03 14:55:20,026 44k INFO Saving model and optimizer state at iteration 106 to ./logs/44k/D_7500.pth
|
554 |
+
2023-05-03 14:55:20,610 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7300.pth
|
555 |
+
2023-05-03 14:55:20,639 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7300.pth
|
556 |
+
2023-05-03 14:55:38,546 44k INFO ====> Epoch: 106, cost 56.63 s
|
557 |
+
2023-05-03 14:56:25,232 44k INFO ====> Epoch: 107, cost 46.69 s
|
558 |
+
2023-05-03 14:56:30,729 44k INFO Train Epoch: 108 [3%]
|
559 |
+
2023-05-03 14:56:30,730 44k INFO Losses: [2.4331343173980713, 2.1562960147857666, 10.360976219177246, 20.71307945251465, 1.2649202346801758], step: 7600, lr: 9.867132229656573e-05, reference_loss: 36.92840576171875
|
560 |
+
2023-05-03 14:56:35,983 44k INFO Saving model and optimizer state at iteration 108 to ./logs/44k/G_7600.pth
|
561 |
+
2023-05-03 14:56:37,594 44k INFO Saving model and optimizer state at iteration 108 to ./logs/44k/D_7600.pth
|
562 |
+
2023-05-03 14:56:38,147 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7400.pth
|
563 |
+
2023-05-03 14:56:38,199 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7400.pth
|
564 |
+
2023-05-03 14:57:19,302 44k INFO ====> Epoch: 108, cost 54.07 s
|
565 |
+
2023-05-03 14:57:42,227 44k INFO Train Epoch: 109 [44%]
|
566 |
+
2023-05-03 14:57:42,228 44k INFO Losses: [2.3168601989746094, 2.5806796550750732, 9.585286140441895, 20.979799270629883, 1.159220814704895], step: 7700, lr: 9.865898838127865e-05, reference_loss: 36.62184524536133
|
567 |
+
2023-05-03 14:57:47,470 44k INFO Saving model and optimizer state at iteration 109 to ./logs/44k/G_7700.pth
|
568 |
+
2023-05-03 14:57:49,300 44k INFO Saving model and optimizer state at iteration 109 to ./logs/44k/D_7700.pth
|
569 |
+
2023-05-03 14:57:49,857 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7500.pth
|
570 |
+
2023-05-03 14:57:49,899 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7500.pth
|
571 |
+
2023-05-03 14:58:13,625 44k INFO ====> Epoch: 109, cost 54.32 s
|
572 |
+
2023-05-03 14:58:54,066 44k INFO Train Epoch: 110 [85%]
|
573 |
+
2023-05-03 14:58:54,067 44k INFO Losses: [2.639904260635376, 2.3363561630249023, 9.849010467529297, 19.11192512512207, 1.1576781272888184], step: 7800, lr: 9.864665600773098e-05, reference_loss: 35.094871520996094
|
574 |
+
2023-05-03 14:58:59,104 44k INFO Saving model and optimizer state at iteration 110 to ./logs/44k/G_7800.pth
|
575 |
+
2023-05-03 14:59:00,792 44k INFO Saving model and optimizer state at iteration 110 to ./logs/44k/D_7800.pth
|
576 |
+
2023-05-03 14:59:01,344 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7600.pth
|
577 |
+
2023-05-03 14:59:01,372 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7600.pth
|
578 |
+
2023-05-03 14:59:07,747 44k INFO ====> Epoch: 110, cost 54.12 s
|
579 |
+
2023-05-03 14:59:59,651 44k INFO ====> Epoch: 111, cost 51.90 s
|
580 |
+
2023-05-03 15:00:14,927 44k INFO Train Epoch: 112 [25%]
|
581 |
+
2023-05-03 15:00:14,928 44k INFO Losses: [2.4285850524902344, 2.606787919998169, 11.0058012008667, 19.486034393310547, 0.8991149663925171], step: 7900, lr: 9.862199588508305e-05, reference_loss: 36.42632293701172
|
582 |
+
2023-05-03 15:00:20,191 44k INFO Saving model and optimizer state at iteration 112 to ./logs/44k/G_7900.pth
|
583 |
+
2023-05-03 15:00:21,924 44k INFO Saving model and optimizer state at iteration 112 to ./logs/44k/D_7900.pth
|
584 |
+
2023-05-03 15:00:22,486 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7700.pth
|
585 |
+
2023-05-03 15:00:22,538 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7700.pth
|
586 |
+
2023-05-03 15:00:54,314 44k INFO ====> Epoch: 112, cost 54.66 s
|
587 |
+
2023-05-03 15:01:27,150 44k INFO Train Epoch: 113 [66%]
|
588 |
+
2023-05-03 15:01:27,151 44k INFO Losses: [2.523535966873169, 2.58498477935791, 7.795136451721191, 22.24933624267578, 1.2834935188293457], step: 8000, lr: 9.86096681355974e-05, reference_loss: 36.436485290527344
|
589 |
+
2023-05-03 15:01:32,472 44k INFO Saving model and optimizer state at iteration 113 to ./logs/44k/G_8000.pth
|
590 |
+
2023-05-03 15:01:34,307 44k INFO Saving model and optimizer state at iteration 113 to ./logs/44k/D_8000.pth
|
591 |
+
2023-05-03 15:01:34,875 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7800.pth
|
592 |
+
2023-05-03 15:01:34,917 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7800.pth
|
593 |
+
2023-05-03 15:01:49,136 44k INFO ====> Epoch: 113, cost 54.82 s
|
594 |
+
2023-05-03 15:02:36,217 44k INFO ====> Epoch: 114, cost 47.08 s
|
595 |
+
2023-05-03 15:02:43,597 44k INFO Train Epoch: 115 [7%]
|
596 |
+
2023-05-03 15:02:43,598 44k INFO Losses: [2.389103889465332, 2.183370351791382, 8.995611190795898, 17.36598014831543, 0.7324305176734924], step: 8100, lr: 9.858501725933955e-05, reference_loss: 31.6664981842041
|
597 |
+
2023-05-03 15:02:48,652 44k INFO Saving model and optimizer state at iteration 115 to ./logs/44k/G_8100.pth
|
598 |
+
2023-05-03 15:02:50,350 44k INFO Saving model and optimizer state at iteration 115 to ./logs/44k/D_8100.pth
|
599 |
+
2023-05-03 15:02:50,908 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_7900.pth
|
600 |
+
2023-05-03 15:02:50,940 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_7900.pth
|
601 |
+
2023-05-03 15:03:30,855 44k INFO ====> Epoch: 115, cost 54.64 s
|
602 |
+
2023-05-03 15:03:55,782 44k INFO Train Epoch: 116 [48%]
|
603 |
+
2023-05-03 15:03:55,783 44k INFO Losses: [2.1408324241638184, 2.1866962909698486, 9.188668251037598, 22.545970916748047, 1.038486123085022], step: 8200, lr: 9.857269413218213e-05, reference_loss: 37.10065460205078
|
604 |
+
2023-05-03 15:04:00,808 44k INFO Saving model and optimizer state at iteration 116 to ./logs/44k/G_8200.pth
|
605 |
+
2023-05-03 15:04:02,446 44k INFO Saving model and optimizer state at iteration 116 to ./logs/44k/D_8200.pth
|
606 |
+
2023-05-03 15:04:03,008 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8000.pth
|
607 |
+
2023-05-03 15:04:03,060 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8000.pth
|
608 |
+
2023-05-03 15:04:25,220 44k INFO ====> Epoch: 116, cost 54.36 s
|
609 |
+
2023-05-03 15:05:07,296 44k INFO Train Epoch: 117 [89%]
|
610 |
+
2023-05-03 15:05:07,297 44k INFO Losses: [2.123523712158203, 2.3391237258911133, 10.087421417236328, 21.18076515197754, 1.0259798765182495], step: 8300, lr: 9.85603725454156e-05, reference_loss: 36.756813049316406
|
611 |
+
2023-05-03 15:05:12,704 44k INFO Saving model and optimizer state at iteration 117 to ./logs/44k/G_8300.pth
|
612 |
+
2023-05-03 15:05:14,431 44k INFO Saving model and optimizer state at iteration 117 to ./logs/44k/D_8300.pth
|
613 |
+
2023-05-03 15:05:14,990 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8100.pth
|
614 |
+
2023-05-03 15:05:15,033 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8100.pth
|
615 |
+
2023-05-03 15:05:19,669 44k INFO ====> Epoch: 117, cost 54.45 s
|
616 |
+
2023-05-03 15:06:06,258 44k INFO ====> Epoch: 118, cost 46.59 s
|
617 |
+
2023-05-03 15:06:23,373 44k INFO Train Epoch: 119 [30%]
|
618 |
+
2023-05-03 15:06:23,374 44k INFO Losses: [2.2387447357177734, 2.410027027130127, 12.253347396850586, 22.95694923400879, 0.9085140228271484], step: 8400, lr: 9.853573399228505e-05, reference_loss: 40.76758575439453
|
619 |
+
2023-05-03 15:06:28,624 44k INFO Saving model and optimizer state at iteration 119 to ./logs/44k/G_8400.pth
|
620 |
+
2023-05-03 15:06:30,293 44k INFO Saving model and optimizer state at iteration 119 to ./logs/44k/D_8400.pth
|
621 |
+
2023-05-03 15:06:30,852 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8200.pth
|
622 |
+
2023-05-03 15:06:30,883 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8200.pth
|
623 |
+
2023-05-03 15:07:00,915 44k INFO ====> Epoch: 119, cost 54.66 s
|
624 |
+
2023-05-03 15:07:35,666 44k INFO Train Epoch: 120 [70%]
|
625 |
+
2023-05-03 15:07:35,667 44k INFO Losses: [2.337846279144287, 2.437479019165039, 10.248119354248047, 18.765079498291016, 0.8664497137069702], step: 8500, lr: 9.8523417025536e-05, reference_loss: 34.65497589111328
|
626 |
+
2023-05-03 15:07:40,791 44k INFO Saving model and optimizer state at iteration 120 to ./logs/44k/G_8500.pth
|
627 |
+
2023-05-03 15:07:42,553 44k INFO Saving model and optimizer state at iteration 120 to ./logs/44k/D_8500.pth
|
628 |
+
2023-05-03 15:07:43,115 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8300.pth
|
629 |
+
2023-05-03 15:07:43,168 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8300.pth
|
630 |
+
2023-05-03 15:07:55,625 44k INFO ====> Epoch: 120, cost 54.71 s
|
631 |
+
2023-05-03 15:08:47,327 44k INFO ====> Epoch: 121, cost 51.70 s
|
632 |
+
2023-05-03 15:08:56,353 44k INFO Train Epoch: 122 [11%]
|
633 |
+
2023-05-03 15:08:56,354 44k INFO Losses: [2.2986395359039307, 2.452651262283325, 10.624441146850586, 18.602886199951172, 1.0152174234390259], step: 8600, lr: 9.8498787710708e-05, reference_loss: 34.99383544921875
|
634 |
+
2023-05-03 15:09:01,490 44k INFO Saving model and optimizer state at iteration 122 to ./logs/44k/G_8600.pth
|
635 |
+
2023-05-03 15:09:03,203 44k INFO Saving model and optimizer state at iteration 122 to ./logs/44k/D_8600.pth
|
636 |
+
2023-05-03 15:09:03,760 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8400.pth
|
637 |
+
2023-05-03 15:09:03,802 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8400.pth
|
638 |
+
2023-05-03 15:09:41,161 44k INFO ====> Epoch: 122, cost 53.83 s
|
639 |
+
2023-05-03 15:10:07,424 44k INFO Train Epoch: 123 [52%]
|
640 |
+
2023-05-03 15:10:07,424 44k INFO Losses: [2.413036823272705, 2.3398640155792236, 10.38782024383545, 20.648679733276367, 0.8482047319412231], step: 8700, lr: 9.848647536224416e-05, reference_loss: 36.63760757446289
|
641 |
+
2023-05-03 15:10:12,546 44k INFO Saving model and optimizer state at iteration 123 to ./logs/44k/G_8700.pth
|
642 |
+
2023-05-03 15:10:14,214 44k INFO Saving model and optimizer state at iteration 123 to ./logs/44k/D_8700.pth
|
643 |
+
2023-05-03 15:10:14,772 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8500.pth
|
644 |
+
2023-05-03 15:10:14,803 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8500.pth
|
645 |
+
2023-05-03 15:10:34,644 44k INFO ====> Epoch: 123, cost 53.48 s
|
646 |
+
2023-05-03 15:11:18,669 44k INFO Train Epoch: 124 [93%]
|
647 |
+
2023-05-03 15:11:18,669 44k INFO Losses: [2.443486213684082, 2.4561853408813477, 8.95302677154541, 21.11248207092285, 1.0387442111968994], step: 8800, lr: 9.847416455282387e-05, reference_loss: 36.00392532348633
|
648 |
+
2023-05-03 15:11:23,885 44k INFO Saving model and optimizer state at iteration 124 to ./logs/44k/G_8800.pth
|
649 |
+
2023-05-03 15:11:25,575 44k INFO Saving model and optimizer state at iteration 124 to ./logs/44k/D_8800.pth
|
650 |
+
2023-05-03 15:11:26,127 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8600.pth
|
651 |
+
2023-05-03 15:11:26,179 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8600.pth
|
652 |
+
2023-05-03 15:11:28,960 44k INFO ====> Epoch: 124, cost 54.32 s
|
653 |
+
2023-05-03 15:12:15,260 44k INFO ====> Epoch: 125, cost 46.30 s
|
654 |
+
2023-05-03 15:12:34,036 44k INFO Train Epoch: 126 [34%]
|
655 |
+
2023-05-03 15:12:34,036 44k INFO Losses: [2.5524790287017822, 2.10302734375, 6.694677352905273, 18.45949935913086, 1.0567528009414673], step: 8900, lr: 9.84495475503445e-05, reference_loss: 30.866436004638672
|
656 |
+
2023-05-03 15:12:39,212 44k INFO Saving model and optimizer state at iteration 126 to ./logs/44k/G_8900.pth
|
657 |
+
2023-05-03 15:12:40,917 44k INFO Saving model and optimizer state at iteration 126 to ./logs/44k/D_8900.pth
|
658 |
+
2023-05-03 15:12:41,479 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8700.pth
|
659 |
+
2023-05-03 15:12:41,526 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8700.pth
|
660 |
+
2023-05-03 15:13:09,584 44k INFO ====> Epoch: 126, cost 54.32 s
|
661 |
+
2023-05-03 15:13:45,650 44k INFO Train Epoch: 127 [75%]
|
662 |
+
2023-05-03 15:13:45,650 44k INFO Losses: [2.4279212951660156, 2.272848606109619, 7.698683738708496, 22.946094512939453, 1.227624535560608], step: 9000, lr: 9.84372413569007e-05, reference_loss: 36.57317352294922
|
663 |
+
2023-05-03 15:13:50,682 44k INFO Saving model and optimizer state at iteration 127 to ./logs/44k/G_9000.pth
|
664 |
+
2023-05-03 15:13:52,264 44k INFO Saving model and optimizer state at iteration 127 to ./logs/44k/D_9000.pth
|
665 |
+
2023-05-03 15:13:52,825 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8800.pth
|
666 |
+
2023-05-03 15:13:52,855 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8800.pth
|
667 |
+
2023-05-03 15:14:03,357 44k INFO ====> Epoch: 127, cost 53.77 s
|
668 |
+
2023-05-03 15:14:50,208 44k INFO ====> Epoch: 128, cost 46.85 s
|
669 |
+
2023-05-03 15:15:01,752 44k INFO Train Epoch: 129 [15%]
|
670 |
+
2023-05-03 15:15:01,753 44k INFO Losses: [2.2435436248779297, 2.89982271194458, 11.728398323059082, 20.402381896972656, 1.4633480310440063], step: 9100, lr: 9.841263358464336e-05, reference_loss: 38.73749542236328
|
671 |
+
2023-05-03 15:15:07,343 44k INFO Saving model and optimizer state at iteration 129 to ./logs/44k/G_9100.pth
|
672 |
+
2023-05-03 15:15:09,302 44k INFO Saving model and optimizer state at iteration 129 to ./logs/44k/D_9100.pth
|
673 |
+
2023-05-03 15:15:09,957 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_8900.pth
|
674 |
+
2023-05-03 15:15:09,957 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_8900.pth
|
675 |
+
2023-05-03 15:15:45,576 44k INFO ====> Epoch: 129, cost 55.37 s
|
676 |
+
2023-05-03 15:16:13,902 44k INFO Train Epoch: 130 [56%]
|
677 |
+
2023-05-03 15:16:13,903 44k INFO Losses: [2.409078359603882, 2.190520763397217, 8.512211799621582, 19.713029861450195, 1.1344952583312988], step: 9200, lr: 9.840033200544528e-05, reference_loss: 33.95933532714844
|
678 |
+
2023-05-03 15:16:19,095 44k INFO Saving model and optimizer state at iteration 130 to ./logs/44k/G_9200.pth
|
679 |
+
2023-05-03 15:16:20,756 44k INFO Saving model and optimizer state at iteration 130 to ./logs/44k/D_9200.pth
|
680 |
+
2023-05-03 15:16:21,317 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9000.pth
|
681 |
+
2023-05-03 15:16:21,365 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9000.pth
|
682 |
+
2023-05-03 15:16:39,759 44k INFO ====> Epoch: 130, cost 54.18 s
|
683 |
+
2023-05-03 15:17:25,441 44k INFO Train Epoch: 131 [97%]
|
684 |
+
2023-05-03 15:17:25,442 44k INFO Losses: [2.3736002445220947, 2.522505283355713, 9.207528114318848, 20.819185256958008, 1.211393117904663], step: 9300, lr: 9.838803196394459e-05, reference_loss: 36.134212493896484
|
685 |
+
2023-05-03 15:17:30,582 44k INFO Saving model and optimizer state at iteration 131 to ./logs/44k/G_9300.pth
|
686 |
+
2023-05-03 15:17:32,333 44k INFO Saving model and optimizer state at iteration 131 to ./logs/44k/D_9300.pth
|
687 |
+
2023-05-03 15:17:32,921 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9100.pth
|
688 |
+
2023-05-03 15:17:32,973 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9100.pth
|
689 |
+
2023-05-03 15:17:34,031 44k INFO ====> Epoch: 131, cost 54.27 s
|
690 |
+
2023-05-03 15:18:20,663 44k INFO ====> Epoch: 132, cost 46.63 s
|
691 |
+
2023-05-03 15:18:41,162 44k INFO Train Epoch: 133 [38%]
|
692 |
+
2023-05-03 15:18:41,162 44k INFO Losses: [2.4086642265319824, 2.2884583473205566, 7.530864715576172, 20.83812141418457, 1.2712756395339966], step: 9400, lr: 9.836343649326659e-05, reference_loss: 34.33738327026367
|
693 |
+
2023-05-03 15:18:46,365 44k INFO Saving model and optimizer state at iteration 133 to ./logs/44k/G_9400.pth
|
694 |
+
2023-05-03 15:18:48,073 44k INFO Saving model and optimizer state at iteration 133 to ./logs/44k/D_9400.pth
|
695 |
+
2023-05-03 15:18:48,642 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9200.pth
|
696 |
+
2023-05-03 15:18:48,694 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9200.pth
|
697 |
+
2023-05-03 15:19:14,844 44k INFO ====> Epoch: 133, cost 54.18 s
|
698 |
+
2023-05-03 15:19:53,005 44k INFO Train Epoch: 134 [79%]
|
699 |
+
2023-05-03 15:19:53,006 44k INFO Losses: [2.2663352489471436, 2.593853235244751, 9.784102439880371, 20.418903350830078, 1.4837355613708496], step: 9500, lr: 9.835114106370493e-05, reference_loss: 36.54692840576172
|
700 |
+
2023-05-03 15:19:58,125 44k INFO Saving model and optimizer state at iteration 134 to ./logs/44k/G_9500.pth
|
701 |
+
2023-05-03 15:19:59,861 44k INFO Saving model and optimizer state at iteration 134 to ./logs/44k/D_9500.pth
|
702 |
+
2023-05-03 15:20:00,423 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9300.pth
|
703 |
+
2023-05-03 15:20:00,476 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9300.pth
|
704 |
+
2023-05-03 15:20:09,180 44k INFO ====> Epoch: 134, cost 54.34 s
|
705 |
+
2023-05-03 15:20:56,127 44k INFO ====> Epoch: 135, cost 46.95 s
|
706 |
+
2023-05-03 15:21:08,711 44k INFO Train Epoch: 136 [20%]
|
707 |
+
2023-05-03 15:21:08,711 44k INFO Losses: [2.484848976135254, 2.1545777320861816, 12.452383995056152, 19.72847557067871, 1.328551173210144], step: 9600, lr: 9.832655481517557e-05, reference_loss: 38.14883804321289
|
708 |
+
2023-05-03 15:21:13,958 44k INFO Saving model and optimizer state at iteration 136 to ./logs/44k/G_9600.pth
|
709 |
+
2023-05-03 15:21:15,541 44k INFO Saving model and optimizer state at iteration 136 to ./logs/44k/D_9600.pth
|
710 |
+
2023-05-03 15:21:16,118 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9400.pth
|
711 |
+
2023-05-03 15:21:16,170 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9400.pth
|
712 |
+
2023-05-03 15:21:49,828 44k INFO ====> Epoch: 136, cost 53.70 s
|
713 |
+
2023-05-03 15:22:19,843 44k INFO Train Epoch: 137 [61%]
|
714 |
+
2023-05-03 15:22:19,844 44k INFO Losses: [2.525221824645996, 2.226086378097534, 9.078721046447754, 22.079233169555664, 0.9952948093414307], step: 9700, lr: 9.831426399582366e-05, reference_loss: 36.90456008911133
|
715 |
+
2023-05-03 15:22:24,949 44k INFO Saving model and optimizer state at iteration 137 to ./logs/44k/G_9700.pth
|
716 |
+
2023-05-03 15:22:26,787 44k INFO Saving model and optimizer state at iteration 137 to ./logs/44k/D_9700.pth
|
717 |
+
2023-05-03 15:22:27,360 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9500.pth
|
718 |
+
2023-05-03 15:22:27,412 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9500.pth
|
719 |
+
2023-05-03 15:22:43,723 44k INFO ====> Epoch: 137, cost 53.89 s
|
720 |
+
2023-05-03 15:23:30,200 44k INFO ====> Epoch: 138, cost 46.48 s
|
721 |
+
2023-05-03 15:23:34,926 44k INFO Train Epoch: 139 [1%]
|
722 |
+
2023-05-03 15:23:34,926 44k INFO Losses: [2.4429895877838135, 2.154385805130005, 8.6116943359375, 20.738656997680664, 1.326363205909729], step: 9800, lr: 9.828968696598508e-05, reference_loss: 35.27408981323242
|
723 |
+
2023-05-03 15:23:39,999 44k INFO Saving model and optimizer state at iteration 139 to ./logs/44k/G_9800.pth
|
724 |
+
2023-05-03 15:23:41,610 44k INFO Saving model and optimizer state at iteration 139 to ./logs/44k/D_9800.pth
|
725 |
+
2023-05-03 15:23:42,172 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9600.pth
|
726 |
+
2023-05-03 15:23:42,225 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9600.pth
|
727 |
+
2023-05-03 15:24:23,989 44k INFO ====> Epoch: 139, cost 53.79 s
|
728 |
+
2023-05-03 15:24:46,199 44k INFO Train Epoch: 140 [42%]
|
729 |
+
2023-05-03 15:24:46,200 44k INFO Losses: [2.2960493564605713, 2.411436080932617, 8.397132873535156, 22.00579833984375, 1.3239285945892334], step: 9900, lr: 9.827740075511432e-05, reference_loss: 36.43434524536133
|
730 |
+
2023-05-03 15:24:51,236 44k INFO Saving model and optimizer state at iteration 140 to ./logs/44k/G_9900.pth
|
731 |
+
2023-05-03 15:24:52,957 44k INFO Saving model and optimizer state at iteration 140 to ./logs/44k/D_9900.pth
|
732 |
+
2023-05-03 15:24:53,546 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9700.pth
|
733 |
+
2023-05-03 15:24:53,600 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9700.pth
|
734 |
+
2023-05-03 15:25:18,300 44k INFO ====> Epoch: 140, cost 54.31 s
|
735 |
+
2023-05-03 15:25:58,385 44k INFO Train Epoch: 141 [83%]
|
736 |
+
2023-05-03 15:25:58,385 44k INFO Losses: [2.2452259063720703, 2.455890655517578, 9.378299713134766, 23.54975128173828, 1.2288228273391724], step: 10000, lr: 9.826511608001993e-05, reference_loss: 38.85799026489258
|
737 |
+
2023-05-03 15:26:03,556 44k INFO Saving model and optimizer state at iteration 141 to ./logs/44k/G_10000.pth
|
738 |
+
2023-05-03 15:26:05,260 44k INFO Saving model and optimizer state at iteration 141 to ./logs/44k/D_10000.pth
|
739 |
+
2023-05-03 15:26:05,837 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9800.pth
|
740 |
+
2023-05-03 15:26:05,890 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9800.pth
|
741 |
+
2023-05-03 15:26:12,950 44k INFO ====> Epoch: 141, cost 54.65 s
|
742 |
+
2023-05-03 15:26:59,982 44k INFO ====> Epoch: 142, cost 47.03 s
|
743 |
+
2023-05-03 15:27:14,406 44k INFO Train Epoch: 143 [24%]
|
744 |
+
2023-05-03 15:27:14,407 44k INFO Losses: [2.392941951751709, 2.2666730880737305, 7.481887340545654, 21.5576171875, 1.1281098127365112], step: 10100, lr: 9.824055133639235e-05, reference_loss: 34.82722854614258
|
745 |
+
2023-05-03 15:27:19,475 44k INFO Saving model and optimizer state at iteration 143 to ./logs/44k/G_10100.pth
|
746 |
+
2023-05-03 15:27:21,296 44k INFO Saving model and optimizer state at iteration 143 to ./logs/44k/D_10100.pth
|
747 |
+
2023-05-03 15:27:21,896 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_9900.pth
|
748 |
+
2023-05-03 15:27:21,948 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_9900.pth
|
749 |
+
2023-05-03 15:27:53,922 44k INFO ====> Epoch: 143, cost 53.94 s
|
750 |
+
2023-05-03 15:28:26,261 44k INFO Train Epoch: 144 [65%]
|
751 |
+
2023-05-03 15:28:26,262 44k INFO Losses: [2.138638973236084, 2.2958712577819824, 12.25405502319336, 19.357269287109375, 0.7592658996582031], step: 10200, lr: 9.822827126747529e-05, reference_loss: 36.80509948730469
|
752 |
+
2023-05-03 15:28:31,318 44k INFO Saving model and optimizer state at iteration 144 to ./logs/44k/G_10200.pth
|
753 |
+
2023-05-03 15:28:33,087 44k INFO Saving model and optimizer state at iteration 144 to ./logs/44k/D_10200.pth
|
754 |
+
2023-05-03 15:28:33,666 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10000.pth
|
755 |
+
2023-05-03 15:28:33,718 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10000.pth
|
756 |
+
2023-05-03 15:28:48,562 44k INFO ====> Epoch: 144, cost 54.64 s
|
757 |
+
2023-05-03 15:29:35,378 44k INFO ====> Epoch: 145, cost 46.82 s
|
758 |
+
2023-05-03 15:29:42,058 44k INFO Train Epoch: 146 [6%]
|
759 |
+
2023-05-03 15:29:42,059 44k INFO Losses: [2.4405500888824463, 2.3585987091064453, 7.771078586578369, 21.579200744628906, 1.055567979812622], step: 10300, lr: 9.820371573447515e-05, reference_loss: 35.20499801635742
|
760 |
+
2023-05-03 15:29:47,198 44k INFO Saving model and optimizer state at iteration 146 to ./logs/44k/G_10300.pth
|
761 |
+
2023-05-03 15:29:48,895 44k INFO Saving model and optimizer state at iteration 146 to ./logs/44k/D_10300.pth
|
762 |
+
2023-05-03 15:29:49,470 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10100.pth
|
763 |
+
2023-05-03 15:29:49,522 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10100.pth
|
764 |
+
2023-05-03 15:30:29,636 44k INFO ====> Epoch: 146, cost 54.26 s
|
765 |
+
2023-05-03 15:30:53,921 44k INFO Train Epoch: 147 [46%]
|
766 |
+
2023-05-03 15:30:53,922 44k INFO Losses: [2.4801368713378906, 2.6744041442871094, 10.43703556060791, 19.093036651611328, 0.8212056159973145], step: 10400, lr: 9.819144027000834e-05, reference_loss: 35.50581741333008
|
767 |
+
2023-05-03 15:30:59,066 44k INFO Saving model and optimizer state at iteration 147 to ./logs/44k/G_10400.pth
|
768 |
+
2023-05-03 15:31:00,779 44k INFO Saving model and optimizer state at iteration 147 to ./logs/44k/D_10400.pth
|
769 |
+
2023-05-03 15:31:01,355 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10200.pth
|
770 |
+
2023-05-03 15:31:01,407 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10200.pth
|
771 |
+
2023-05-03 15:31:24,078 44k INFO ====> Epoch: 147, cost 54.44 s
|
772 |
+
2023-05-03 15:32:10,012 44k INFO Train Epoch: 148 [87%]
|
773 |
+
2023-05-03 15:32:10,013 44k INFO Losses: [2.3031187057495117, 2.331038236618042, 12.805354118347168, 21.40829849243164, 0.8880621194839478], step: 10500, lr: 9.817916633997459e-05, reference_loss: 39.735870361328125
|
774 |
+
2023-05-03 15:32:15,130 44k INFO Saving model and optimizer state at iteration 148 to ./logs/44k/G_10500.pth
|
775 |
+
2023-05-03 15:32:16,899 44k INFO Saving model and optimizer state at iteration 148 to ./logs/44k/D_10500.pth
|
776 |
+
2023-05-03 15:32:17,477 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10300.pth
|
777 |
+
2023-05-03 15:32:17,529 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10300.pth
|
778 |
+
2023-05-03 15:32:22,759 44k INFO ====> Epoch: 148, cost 58.68 s
|
779 |
+
2023-05-03 15:33:09,200 44k INFO ====> Epoch: 149, cost 46.44 s
|
780 |
+
2023-05-03 15:33:25,713 44k INFO Train Epoch: 150 [28%]
|
781 |
+
2023-05-03 15:33:25,714 44k INFO Losses: [2.7253715991973877, 2.3599607944488525, 5.7014360427856445, 17.386552810668945, 1.0759114027023315], step: 10600, lr: 9.815462308243906e-05, reference_loss: 29.249231338500977
|
782 |
+
2023-05-03 15:33:30,700 44k INFO Saving model and optimizer state at iteration 150 to ./logs/44k/G_10600.pth
|
783 |
+
2023-05-03 15:33:32,395 44k INFO Saving model and optimizer state at iteration 150 to ./logs/44k/D_10600.pth
|
784 |
+
2023-05-03 15:33:32,964 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10400.pth
|
785 |
+
2023-05-03 15:33:33,016 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10400.pth
|
786 |
+
2023-05-03 15:34:03,304 44k INFO ====> Epoch: 150, cost 54.10 s
|
787 |
+
2023-05-03 15:34:37,855 44k INFO Train Epoch: 151 [69%]
|
788 |
+
2023-05-03 15:34:37,855 44k INFO Losses: [2.3957998752593994, 2.5786211490631104, 8.307215690612793, 20.157119750976562, 0.9237015843391418], step: 10700, lr: 9.814235375455375e-05, reference_loss: 34.36246109008789
|
789 |
+
2023-05-03 15:34:43,152 44k INFO Saving model and optimizer state at iteration 151 to ./logs/44k/G_10700.pth
|
790 |
+
2023-05-03 15:34:44,852 44k INFO Saving model and optimizer state at iteration 151 to ./logs/44k/D_10700.pth
|
791 |
+
2023-05-03 15:34:45,424 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10500.pth
|
792 |
+
2023-05-03 15:34:45,476 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10500.pth
|
793 |
+
2023-05-03 15:34:58,329 44k INFO ====> Epoch: 151, cost 55.02 s
|
794 |
+
2023-05-03 15:35:45,281 44k INFO ====> Epoch: 152, cost 46.95 s
|
795 |
+
2023-05-03 15:35:53,820 44k INFO Train Epoch: 153 [10%]
|
796 |
+
2023-05-03 15:35:53,821 44k INFO Losses: [2.449925422668457, 2.4277830123901367, 6.188196659088135, 20.815114974975586, 0.7682157158851624], step: 10800, lr: 9.811781969958938e-05, reference_loss: 32.649234771728516
|
797 |
+
2023-05-03 15:35:59,223 44k INFO Saving model and optimizer state at iteration 153 to ./logs/44k/G_10800.pth
|
798 |
+
2023-05-03 15:36:00,869 44k INFO Saving model and optimizer state at iteration 153 to ./logs/44k/D_10800.pth
|
799 |
+
2023-05-03 15:36:01,450 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10600.pth
|
800 |
+
2023-05-03 15:36:01,503 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10600.pth
|
801 |
+
2023-05-03 15:36:40,018 44k INFO ====> Epoch: 153, cost 54.74 s
|
802 |
+
2023-05-03 15:37:05,909 44k INFO Train Epoch: 154 [51%]
|
803 |
+
2023-05-03 15:37:05,910 44k INFO Losses: [2.2990469932556152, 2.5312039852142334, 8.215372085571289, 19.852989196777344, 1.0274193286895752], step: 10900, lr: 9.810555497212693e-05, reference_loss: 33.92603302001953
|
804 |
+
2023-05-03 15:37:10,984 44k INFO Saving model and optimizer state at iteration 154 to ./logs/44k/G_10900.pth
|
805 |
+
2023-05-03 15:37:12,868 44k INFO Saving model and optimizer state at iteration 154 to ./logs/44k/D_10900.pth
|
806 |
+
2023-05-03 15:37:13,446 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10700.pth
|
807 |
+
2023-05-03 15:37:13,498 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10700.pth
|
808 |
+
2023-05-03 15:37:34,031 44k INFO ====> Epoch: 154, cost 54.01 s
|
809 |
+
2023-05-03 15:38:17,770 44k INFO Train Epoch: 155 [92%]
|
810 |
+
2023-05-03 15:38:17,771 44k INFO Losses: [2.347559928894043, 2.624699592590332, 10.70503044128418, 18.965770721435547, 0.6355746984481812], step: 11000, lr: 9.809329177775541e-05, reference_loss: 35.27863311767578
|
811 |
+
2023-05-03 15:38:23,015 44k INFO Saving model and optimizer state at iteration 155 to ./logs/44k/G_11000.pth
|
812 |
+
2023-05-03 15:38:24,698 44k INFO Saving model and optimizer state at iteration 155 to ./logs/44k/D_11000.pth
|
813 |
+
2023-05-03 15:38:25,283 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10800.pth
|
814 |
+
2023-05-03 15:38:25,336 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10800.pth
|
815 |
+
2023-05-03 15:38:28,805 44k INFO ====> Epoch: 155, cost 54.77 s
|
816 |
+
2023-05-03 15:39:15,570 44k INFO ====> Epoch: 156, cost 46.76 s
|
817 |
+
2023-05-03 15:39:35,466 44k INFO Train Epoch: 157 [32%]
|
818 |
+
2023-05-03 15:39:35,467 44k INFO Losses: [2.576101303100586, 2.190938949584961, 7.242859840393066, 21.8758602142334, 0.8499348163604736], step: 11100, lr: 9.806876998751865e-05, reference_loss: 34.735694885253906
|
819 |
+
2023-05-03 15:39:40,781 44k INFO Saving model and optimizer state at iteration 157 to ./logs/44k/G_11100.pth
|
820 |
+
2023-05-03 15:39:42,545 44k INFO Saving model and optimizer state at iteration 157 to ./logs/44k/D_11100.pth
|
821 |
+
2023-05-03 15:39:43,126 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_10900.pth
|
822 |
+
2023-05-03 15:39:43,178 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_10900.pth
|
823 |
+
2023-05-03 15:40:13,774 44k INFO ====> Epoch: 157, cost 58.20 s
|
824 |
+
2023-05-03 15:40:49,393 44k INFO Train Epoch: 158 [73%]
|
825 |
+
2023-05-03 15:40:49,394 44k INFO Losses: [2.4354920387268066, 2.526336193084717, 9.373090744018555, 20.170114517211914, 1.5362391471862793], step: 11200, lr: 9.80565113912702e-05, reference_loss: 36.04127502441406
|
826 |
+
2023-05-03 15:40:54,411 44k INFO Saving model and optimizer state at iteration 158 to ./logs/44k/G_11200.pth
|
827 |
+
2023-05-03 15:40:56,229 44k INFO Saving model and optimizer state at iteration 158 to ./logs/44k/D_11200.pth
|
828 |
+
2023-05-03 15:40:56,811 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11000.pth
|
829 |
+
2023-05-03 15:40:56,863 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11000.pth
|
830 |
+
2023-05-03 15:41:07,949 44k INFO ====> Epoch: 158, cost 54.17 s
|
831 |
+
2023-05-03 15:41:55,183 44k INFO ====> Epoch: 159, cost 47.23 s
|
832 |
+
2023-05-03 15:42:05,496 44k INFO Train Epoch: 160 [14%]
|
833 |
+
2023-05-03 15:42:05,497 44k INFO Losses: [2.44669246673584, 2.414104461669922, 9.693013191223145, 20.203325271606445, 1.273841381072998], step: 11300, lr: 9.803199879555537e-05, reference_loss: 36.030975341796875
|
834 |
+
2023-05-03 15:42:10,663 44k INFO Saving model and optimizer state at iteration 160 to ./logs/44k/G_11300.pth
|
835 |
+
2023-05-03 15:42:12,281 44k INFO Saving model and optimizer state at iteration 160 to ./logs/44k/D_11300.pth
|
836 |
+
2023-05-03 15:42:12,863 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11100.pth
|
837 |
+
2023-05-03 15:42:12,915 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11100.pth
|
838 |
+
2023-05-03 15:42:49,118 44k INFO ====> Epoch: 160, cost 53.93 s
|
839 |
+
2023-05-03 15:43:16,997 44k INFO Train Epoch: 161 [55%]
|
840 |
+
2023-05-03 15:43:16,997 44k INFO Losses: [2.445122241973877, 2.363597869873047, 10.572999954223633, 20.042144775390625, 1.0533027648925781], step: 11400, lr: 9.801974479570593e-05, reference_loss: 36.477169036865234
|
841 |
+
2023-05-03 15:43:22,411 44k INFO Saving model and optimizer state at iteration 161 to ./logs/44k/G_11400.pth
|
842 |
+
2023-05-03 15:43:24,192 44k INFO Saving model and optimizer state at iteration 161 to ./logs/44k/D_11400.pth
|
843 |
+
2023-05-03 15:43:24,775 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11200.pth
|
844 |
+
2023-05-03 15:43:24,828 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11200.pth
|
845 |
+
2023-05-03 15:43:45,908 44k INFO ====> Epoch: 161, cost 56.79 s
|
846 |
+
2023-05-03 15:44:31,322 44k INFO Train Epoch: 162 [96%]
|
847 |
+
2023-05-03 15:44:31,323 44k INFO Losses: [2.3155155181884766, 2.432081699371338, 10.127751350402832, 20.838157653808594, 0.8625903725624084], step: 11500, lr: 9.800749232760646e-05, reference_loss: 36.57609939575195
|
848 |
+
2023-05-03 15:44:36,426 44k INFO Saving model and optimizer state at iteration 162 to ./logs/44k/G_11500.pth
|
849 |
+
2023-05-03 15:44:38,226 44k INFO Saving model and optimizer state at iteration 162 to ./logs/44k/D_11500.pth
|
850 |
+
2023-05-03 15:44:38,806 44k INFO .. Free up space by deleting ckpt ./logs/44k/G_11300.pth
|
851 |
+
2023-05-03 15:44:38,858 44k INFO .. Free up space by deleting ckpt ./logs/44k/D_11300.pth
|
852 |
+
2023-05-03 15:44:40,491 44k INFO ====> Epoch: 162, cost 54.58 s
|