dat commited on
Commit
4229c91
1 Parent(s): 0446688
config.json CHANGED
@@ -4,7 +4,7 @@
4
  ],
5
  "attention_probs_dropout_prob": 0.1,
6
  "attention_type": "block_sparse",
7
- "block_size": 128,
8
  "bos_token_id": 1,
9
  "eos_token_id": 2,
10
  "gradient_checkpointing": false,
 
4
  ],
5
  "attention_probs_dropout_prob": 0.1,
6
  "attention_type": "block_sparse",
7
+ "block_size": 64,
8
  "bos_token_id": 1,
9
  "eos_token_id": 2,
10
  "gradient_checkpointing": false,
events.out.tfevents.1626135630.t1v-n-f5c06ea1-w-0.319169.3.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2be9a3fee86650547c5ddda2f6a3e6336bd828bc45c95d87626f6c16425f071
3
+ size 147065
run_mlm_flax.py CHANGED
@@ -699,7 +699,7 @@ if __name__ == "__main__":
699
  train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size // grad_accum_steps)
700
 
701
  # Gather the indexes for creating the batch and do a training step
702
- for step, batch_idx in enumerate(tqdm(train_batch_idx, desc="Training...", position=1,initial=resume_step)):
703
  samples = [train_dataset[int(idx)] for idx in batch_idx]
704
  model_inputs = data_collator(samples, pad_to_multiple_of=16)
705
 
 
699
  train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size // grad_accum_steps)
700
 
701
  # Gather the indexes for creating the batch and do a training step
702
+ for step, batch_idx in enumerate(tqdm(train_batch_idx, desc="Training...", position=1,initial=resume_step // grad_accum_steps )):
703
  samples = [train_dataset[int(idx)] for idx in batch_idx]
704
  model_inputs = data_collator(samples, pad_to_multiple_of=16)
705
 
wandb/debug-internal.log ADDED
@@ -0,0 +1 @@
 
 
1
+ run-20210713_002031-11bfig2u/logs/debug-internal.log
wandb/debug.log ADDED
@@ -0,0 +1 @@
 
 
1
+ run-20210713_002031-11bfig2u/logs/debug.log
wandb/latest-run ADDED
@@ -0,0 +1 @@
 
 
1
+ run-20210713_002031-11bfig2u
wandb/run-20210713_002031-11bfig2u/files/config.yaml ADDED
@@ -0,0 +1,304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ cli_version: 0.10.33
7
+ framework: huggingface
8
+ huggingface_version: 4.9.0.dev0
9
+ is_jupyter_run: false
10
+ is_kaggle_kernel: false
11
+ python_version: 3.8.10
12
+ t:
13
+ 1:
14
+ - 3
15
+ - 11
16
+ 4: 3.8.10
17
+ 5: 0.10.33
18
+ 6: 4.9.0.dev0
19
+ 8:
20
+ - 5
21
+ adafactor:
22
+ desc: null
23
+ value: true
24
+ adam_beta1:
25
+ desc: null
26
+ value: 0.9
27
+ adam_beta2:
28
+ desc: null
29
+ value: 0.98
30
+ adam_epsilon:
31
+ desc: null
32
+ value: 1.0e-08
33
+ cache_dir:
34
+ desc: null
35
+ value: null
36
+ config_name:
37
+ desc: null
38
+ value: ./
39
+ dataloader_drop_last:
40
+ desc: null
41
+ value: false
42
+ dataloader_num_workers:
43
+ desc: null
44
+ value: 0
45
+ dataloader_pin_memory:
46
+ desc: null
47
+ value: true
48
+ dataset_config_name:
49
+ desc: null
50
+ value: null
51
+ dataset_name:
52
+ desc: null
53
+ value: null
54
+ ddp_find_unused_parameters:
55
+ desc: null
56
+ value: null
57
+ debug:
58
+ desc: null
59
+ value: []
60
+ deepspeed:
61
+ desc: null
62
+ value: null
63
+ disable_tqdm:
64
+ desc: null
65
+ value: false
66
+ do_eval:
67
+ desc: null
68
+ value: false
69
+ do_predict:
70
+ desc: null
71
+ value: false
72
+ do_train:
73
+ desc: null
74
+ value: false
75
+ dtype:
76
+ desc: null
77
+ value: bfloat16
78
+ eval_accumulation_steps:
79
+ desc: null
80
+ value: null
81
+ eval_steps:
82
+ desc: null
83
+ value: 92768
84
+ evaluation_strategy:
85
+ desc: null
86
+ value: IntervalStrategy.NO
87
+ fp16:
88
+ desc: null
89
+ value: false
90
+ fp16_backend:
91
+ desc: null
92
+ value: auto
93
+ fp16_full_eval:
94
+ desc: null
95
+ value: false
96
+ fp16_opt_level:
97
+ desc: null
98
+ value: O1
99
+ gradient_accumulation_steps:
100
+ desc: null
101
+ value: 1
102
+ greater_is_better:
103
+ desc: null
104
+ value: null
105
+ group_by_length:
106
+ desc: null
107
+ value: false
108
+ ignore_data_skip:
109
+ desc: null
110
+ value: false
111
+ label_names:
112
+ desc: null
113
+ value: null
114
+ label_smoothing_factor:
115
+ desc: null
116
+ value: 0.0
117
+ learning_rate:
118
+ desc: null
119
+ value: 5.0e-05
120
+ length_column_name:
121
+ desc: null
122
+ value: length
123
+ line_by_line:
124
+ desc: null
125
+ value: false
126
+ load_best_model_at_end:
127
+ desc: null
128
+ value: false
129
+ local_rank:
130
+ desc: null
131
+ value: -1
132
+ log_level:
133
+ desc: null
134
+ value: -1
135
+ log_level_replica:
136
+ desc: null
137
+ value: -1
138
+ log_on_each_node:
139
+ desc: null
140
+ value: true
141
+ logging_dir:
142
+ desc: null
143
+ value: ./runs/Jul13_00-19-44_t1v-n-f5c06ea1-w-0
144
+ logging_first_step:
145
+ desc: null
146
+ value: false
147
+ logging_steps:
148
+ desc: null
149
+ value: 500
150
+ logging_strategy:
151
+ desc: null
152
+ value: IntervalStrategy.STEPS
153
+ lr_scheduler_type:
154
+ desc: null
155
+ value: SchedulerType.LINEAR
156
+ max_grad_norm:
157
+ desc: null
158
+ value: 1.0
159
+ max_seq_length:
160
+ desc: null
161
+ value: 4096
162
+ max_steps:
163
+ desc: null
164
+ value: -1
165
+ metric_for_best_model:
166
+ desc: null
167
+ value: null
168
+ mlm_probability:
169
+ desc: null
170
+ value: 0.15
171
+ model_name_or_path:
172
+ desc: null
173
+ value: null
174
+ model_type:
175
+ desc: null
176
+ value: big_bird
177
+ mp_parameters:
178
+ desc: null
179
+ value: ''
180
+ no_cuda:
181
+ desc: null
182
+ value: false
183
+ num_train_epochs:
184
+ desc: null
185
+ value: 5.0
186
+ output_dir:
187
+ desc: null
188
+ value: ./
189
+ overwrite_cache:
190
+ desc: null
191
+ value: false
192
+ overwrite_output_dir:
193
+ desc: null
194
+ value: true
195
+ pad_to_max_length:
196
+ desc: null
197
+ value: false
198
+ past_index:
199
+ desc: null
200
+ value: -1
201
+ per_device_eval_batch_size:
202
+ desc: null
203
+ value: 2
204
+ per_device_train_batch_size:
205
+ desc: null
206
+ value: 2
207
+ per_gpu_eval_batch_size:
208
+ desc: null
209
+ value: null
210
+ per_gpu_train_batch_size:
211
+ desc: null
212
+ value: null
213
+ prediction_loss_only:
214
+ desc: null
215
+ value: false
216
+ preprocessing_num_workers:
217
+ desc: null
218
+ value: 64
219
+ push_to_hub:
220
+ desc: null
221
+ value: true
222
+ push_to_hub_model_id:
223
+ desc: null
224
+ value: ''
225
+ push_to_hub_organization:
226
+ desc: null
227
+ value: null
228
+ push_to_hub_token:
229
+ desc: null
230
+ value: null
231
+ remove_unused_columns:
232
+ desc: null
233
+ value: true
234
+ report_to:
235
+ desc: null
236
+ value:
237
+ - tensorboard
238
+ - wandb
239
+ resume_from_checkpoint:
240
+ desc: null
241
+ value: null
242
+ run_name:
243
+ desc: null
244
+ value: ./
245
+ save_on_each_node:
246
+ desc: null
247
+ value: false
248
+ save_steps:
249
+ desc: null
250
+ value: 20000
251
+ save_strategy:
252
+ desc: null
253
+ value: IntervalStrategy.STEPS
254
+ save_total_limit:
255
+ desc: null
256
+ value: 5
257
+ seed:
258
+ desc: null
259
+ value: 42
260
+ sharded_ddp:
261
+ desc: null
262
+ value: []
263
+ skip_memory_metrics:
264
+ desc: null
265
+ value: true
266
+ tokenizer_name:
267
+ desc: null
268
+ value: ./
269
+ tpu_metrics_debug:
270
+ desc: null
271
+ value: false
272
+ tpu_num_cores:
273
+ desc: null
274
+ value: null
275
+ train_file:
276
+ desc: null
277
+ value: null
278
+ train_ref_file:
279
+ desc: null
280
+ value: null
281
+ use_fast_tokenizer:
282
+ desc: null
283
+ value: true
284
+ use_legacy_prediction_loop:
285
+ desc: null
286
+ value: false
287
+ validation_file:
288
+ desc: null
289
+ value: null
290
+ validation_ref_file:
291
+ desc: null
292
+ value: null
293
+ validation_split_percentage:
294
+ desc: null
295
+ value: 5
296
+ warmup_ratio:
297
+ desc: null
298
+ value: 0.0
299
+ warmup_steps:
300
+ desc: null
301
+ value: 5000
302
+ weight_decay:
303
+ desc: null
304
+ value: 0.0095
wandb/run-20210713_002031-11bfig2u/files/output.log ADDED
@@ -0,0 +1,389 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
2
+ warnings.warn(
3
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
4
+ warnings.warn(
5
+ Epoch ... (1/5): 0%| | 0/5 [00:00<?, ?it/s]
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
52
+
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
65
+
66
+
67
+
68
+
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+
80
+
81
+
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+
120
+
121
+
122
+
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+ Training...: 1%| | 500/92767 [06:10<14:45:27, 1.74it/s]
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+
170
+
171
+
172
+
173
+
174
+
175
+
176
+
177
+
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+
192
+
193
+
194
+
195
+
196
+
197
+
198
+
199
+
200
+
201
+
202
+
203
+
204
+
205
+
206
+
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+
226
+
227
+
228
+
229
+
230
+
231
+
232
+
233
+
234
+
235
+
236
+
237
+
238
+
239
+
240
+
241
+
242
+
243
+
244
+
245
+
246
+
247
+
248
+
249
+
250
+
251
+
252
+
253
+
254
+
255
+
256
+
257
+
258
+
259
+
260
+
261
+
262
+
263
+
264
+
265
+
266
+
267
+
268
+
269
+
270
+
271
+
272
+
273
+
274
+
275
+
276
+
277
+ Training...: 1%|▏ | 1009/92767 [11:19<11:54:13, 2.14it/s]
278
+
279
+
280
+
281
+
282
+
283
+
284
+
285
+
286
+
287
+
288
+
289
+
290
+
291
+
292
+
293
+
294
+
295
+
296
+
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+
307
+
308
+
309
+
310
+
311
+
312
+
313
+
314
+
315
+
316
+
317
+
318
+
319
+
320
+
321
+
322
+
323
+
324
+
325
+
326
+
327
+
328
+
329
+
330
+
331
+
332
+
333
+
334
+
335
+
336
+
337
+
338
+
339
+
340
+
341
+
342
+
343
+
344
+
345
+
346
+
347
+
348
+
349
+
350
+
351
+
352
+
353
+
354
+
355
+
356
+
357
+
358
+
359
+
360
+
361
+
362
+
363
+
364
+
365
+
366
+
367
+
368
+
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+
379
+
380
+
381
+
382
+
383
+
384
+
385
+
386
+
387
+
388
+
389
+
wandb/run-20210713_002031-11bfig2u/files/requirements.txt ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.13.0
2
+ aiohttp==3.7.4.post0
3
+ astunparse==1.6.3
4
+ async-timeout==3.0.1
5
+ attrs==21.2.0
6
+ cachetools==4.2.2
7
+ certifi==2021.5.30
8
+ chardet==4.0.0
9
+ chex==0.0.8
10
+ click==8.0.1
11
+ configparser==5.0.2
12
+ cycler==0.10.0
13
+ datasets==1.9.1.dev0
14
+ dill==0.3.4
15
+ dm-tree==0.1.6
16
+ docker-pycreds==0.4.0
17
+ filelock==3.0.12
18
+ flatbuffers==1.12
19
+ flax==0.3.4
20
+ fsspec==2021.6.1
21
+ gast==0.4.0
22
+ gitdb==4.0.7
23
+ gitpython==3.1.18
24
+ google-auth-oauthlib==0.4.4
25
+ google-auth==1.32.1
26
+ google-pasta==0.2.0
27
+ grpcio==1.34.1
28
+ h5py==3.1.0
29
+ huggingface-hub==0.0.12
30
+ idna==2.10
31
+ jax==0.2.16
32
+ jaxlib==0.1.68
33
+ joblib==1.0.1
34
+ keras-nightly==2.5.0.dev2021032900
35
+ keras-preprocessing==1.1.2
36
+ kiwisolver==1.3.1
37
+ libtpu-nightly==0.1.dev20210615
38
+ markdown==3.3.4
39
+ matplotlib==3.4.2
40
+ msgpack==1.0.2
41
+ multidict==5.1.0
42
+ multiprocess==0.70.12.2
43
+ numpy==1.19.5
44
+ oauthlib==3.1.1
45
+ opt-einsum==3.3.0
46
+ optax==0.0.9
47
+ packaging==21.0
48
+ pandas==1.3.0
49
+ pathtools==0.1.2
50
+ pillow==8.3.1
51
+ pip==20.0.2
52
+ pkg-resources==0.0.0
53
+ promise==2.3
54
+ protobuf==3.17.3
55
+ psutil==5.8.0
56
+ pyarrow==4.0.1
57
+ pyasn1-modules==0.2.8
58
+ pyasn1==0.4.8
59
+ pyparsing==2.4.7
60
+ python-dateutil==2.8.1
61
+ pytz==2021.1
62
+ pyyaml==5.4.1
63
+ regex==2021.7.6
64
+ requests-oauthlib==1.3.0
65
+ requests==2.25.1
66
+ rsa==4.7.2
67
+ sacremoses==0.0.45
68
+ scipy==1.7.0
69
+ sentry-sdk==1.3.0
70
+ setuptools==44.0.0
71
+ shortuuid==1.0.1
72
+ six==1.15.0
73
+ smmap==4.0.0
74
+ subprocess32==3.5.4
75
+ tensorboard-data-server==0.6.1
76
+ tensorboard-plugin-wit==1.8.0
77
+ tensorboard==2.5.0
78
+ tensorflow-estimator==2.5.0
79
+ tensorflow==2.5.0
80
+ termcolor==1.1.0
81
+ tokenizers==0.10.3
82
+ toolz==0.11.1
83
+ tqdm==4.61.2
84
+ transformers==4.9.0.dev0
85
+ typing-extensions==3.7.4.3
86
+ urllib3==1.26.6
87
+ wandb==0.10.33
88
+ werkzeug==2.0.1
89
+ wheel==0.36.2
90
+ wrapt==1.12.1
91
+ xxhash==2.0.2
92
+ yarl==1.6.3
wandb/run-20210713_002031-11bfig2u/files/wandb-metadata.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-07-13T00:20:33.520986",
5
+ "startedAt": "2021-07-13T00:20:31.453492",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--push_to_hub",
11
+ "--output_dir=./",
12
+ "--model_type=big_bird",
13
+ "--config_name=./",
14
+ "--tokenizer_name=./",
15
+ "--max_seq_length=4096",
16
+ "--weight_decay=0.0095",
17
+ "--warmup_steps=5000",
18
+ "--overwrite_output_dir",
19
+ "--adam_beta1=0.9",
20
+ "--adam_beta2=0.98",
21
+ "--logging_steps=500",
22
+ "--eval_steps=92768",
23
+ "--num_train_epochs=5",
24
+ "--preprocessing_num_workers=64",
25
+ "--save_steps=20000",
26
+ "--adafactor",
27
+ "--learning_rate=5e-5",
28
+ "--per_device_train_batch_size=2",
29
+ "--per_device_eval_batch_size=2",
30
+ "--save_total_limit=5",
31
+ "--dtype=bfloat16"
32
+ ],
33
+ "state": "running",
34
+ "program": "./run_mlm_flax.py",
35
+ "codePath": "run_mlm_flax.py",
36
+ "git": {
37
+ "remote": "https://huggingface.co/flax-community/pino-roberta-base",
38
+ "commit": "044668814aaa4e4a41e779e25084064cca6fe955"
39
+ },
40
+ "email": null,
41
+ "root": "/home/dat/pino-roberta-base",
42
+ "host": "t1v-n-f5c06ea1-w-0",
43
+ "username": "dat",
44
+ "executable": "/home/dat/pino/bin/python"
45
+ }
wandb/run-20210713_002031-11bfig2u/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"training_step": 1000, "learning_rate": 9.999999747378752e-06, "train_loss": 10.829078674316406, "_runtime": 789, "_timestamp": 1626136420, "_step": 1}
wandb/run-20210713_002031-11bfig2u/logs/debug-internal.log ADDED
@@ -0,0 +1,605 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-13 00:20:32,185 INFO MainThread:320742 [internal.py:wandb_internal():88] W&B internal server running at pid: 320742, started at: 2021-07-13 00:20:32.185297
2
+ 2021-07-13 00:20:32,187 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: check_version
3
+ 2021-07-13 00:20:32,187 INFO WriterThread:320742 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/run-11bfig2u.wandb
4
+ 2021-07-13 00:20:32,188 DEBUG SenderThread:320742 [sender.py:send():179] send: header
5
+ 2021-07-13 00:20:32,189 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: check_version
6
+ 2021-07-13 00:20:32,228 DEBUG SenderThread:320742 [sender.py:send():179] send: run
7
+ 2021-07-13 00:20:32,408 INFO SenderThread:320742 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files
8
+ 2021-07-13 00:20:32,409 INFO SenderThread:320742 [sender.py:_start_run_threads():716] run started: 11bfig2u with start time 1626135631
9
+ 2021-07-13 00:20:32,409 DEBUG SenderThread:320742 [sender.py:send():179] send: summary
10
+ 2021-07-13 00:20:32,409 INFO SenderThread:320742 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
11
+ 2021-07-13 00:20:32,409 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: run_start
12
+ 2021-07-13 00:20:33,411 INFO Thread-8 :320742 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/wandb-summary.json
13
+ 2021-07-13 00:20:33,520 DEBUG HandlerThread:320742 [meta.py:__init__():39] meta init
14
+ 2021-07-13 00:20:33,520 DEBUG HandlerThread:320742 [meta.py:__init__():53] meta init done
15
+ 2021-07-13 00:20:33,520 DEBUG HandlerThread:320742 [meta.py:probe():210] probe
16
+ 2021-07-13 00:20:33,522 DEBUG HandlerThread:320742 [meta.py:_setup_git():200] setup git
17
+ 2021-07-13 00:20:33,551 DEBUG HandlerThread:320742 [meta.py:_setup_git():207] setup git done
18
+ 2021-07-13 00:20:33,551 DEBUG HandlerThread:320742 [meta.py:_save_pip():57] save pip
19
+ 2021-07-13 00:20:33,551 DEBUG HandlerThread:320742 [meta.py:_save_pip():71] save pip done
20
+ 2021-07-13 00:20:33,551 DEBUG HandlerThread:320742 [meta.py:probe():252] probe done
21
+ 2021-07-13 00:20:33,554 DEBUG SenderThread:320742 [sender.py:send():179] send: files
22
+ 2021-07-13 00:20:33,554 INFO SenderThread:320742 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
23
+ 2021-07-13 00:20:33,561 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
24
+ 2021-07-13 00:20:33,561 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
25
+ 2021-07-13 00:20:33,690 DEBUG SenderThread:320742 [sender.py:send():179] send: config
26
+ 2021-07-13 00:20:33,691 DEBUG SenderThread:320742 [sender.py:send():179] send: config
27
+ 2021-07-13 00:20:33,691 DEBUG SenderThread:320742 [sender.py:send():179] send: config
28
+ 2021-07-13 00:20:34,001 INFO Thread-11 :320742 [upload_job.py:push():137] Uploaded file /tmp/tmpp780arizwandb/1gon5458-wandb-metadata.json
29
+ 2021-07-13 00:20:34,410 INFO Thread-8 :320742 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
30
+ 2021-07-13 00:20:34,410 INFO Thread-8 :320742 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/wandb-metadata.json
31
+ 2021-07-13 00:20:34,410 INFO Thread-8 :320742 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/requirements.txt
32
+ 2021-07-13 00:20:48,692 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
33
+ 2021-07-13 00:20:48,692 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
34
+ 2021-07-13 00:20:52,417 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
35
+ 2021-07-13 00:21:01,606 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
36
+ 2021-07-13 00:21:03,421 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/config.yaml
37
+ 2021-07-13 00:21:03,825 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
38
+ 2021-07-13 00:21:03,825 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
39
+ 2021-07-13 00:21:18,959 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
40
+ 2021-07-13 00:21:18,959 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
41
+ 2021-07-13 00:21:31,685 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
42
+ 2021-07-13 00:21:34,089 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
43
+ 2021-07-13 00:21:34,090 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
44
+ 2021-07-13 00:21:49,221 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
45
+ 2021-07-13 00:21:49,221 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
46
+ 2021-07-13 00:22:01,760 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
47
+ 2021-07-13 00:22:04,352 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
48
+ 2021-07-13 00:22:04,353 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
49
+ 2021-07-13 00:22:19,487 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
50
+ 2021-07-13 00:22:19,487 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
51
+ 2021-07-13 00:22:24,448 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
52
+ 2021-07-13 00:22:31,839 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
53
+ 2021-07-13 00:22:34,633 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
54
+ 2021-07-13 00:22:34,633 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
55
+ 2021-07-13 00:22:49,777 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
56
+ 2021-07-13 00:22:49,778 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
57
+ 2021-07-13 00:23:01,918 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
58
+ 2021-07-13 00:23:04,908 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
59
+ 2021-07-13 00:23:04,908 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
60
+ 2021-07-13 00:23:20,043 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
61
+ 2021-07-13 00:23:20,043 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
62
+ 2021-07-13 00:23:31,998 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
63
+ 2021-07-13 00:23:35,174 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
64
+ 2021-07-13 00:23:35,174 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
65
+ 2021-07-13 00:23:50,319 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
66
+ 2021-07-13 00:23:50,319 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
67
+ 2021-07-13 00:24:02,076 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
68
+ 2021-07-13 00:24:04,486 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
69
+ 2021-07-13 00:24:05,464 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
70
+ 2021-07-13 00:24:05,464 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
71
+ 2021-07-13 00:24:06,487 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
72
+ 2021-07-13 00:24:08,488 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
73
+ 2021-07-13 00:24:10,489 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
74
+ 2021-07-13 00:24:12,489 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
75
+ 2021-07-13 00:24:14,491 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
76
+ 2021-07-13 00:24:16,491 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
77
+ 2021-07-13 00:24:18,492 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
78
+ 2021-07-13 00:24:20,493 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
79
+ 2021-07-13 00:24:20,641 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
80
+ 2021-07-13 00:24:20,641 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
81
+ 2021-07-13 00:24:22,494 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
82
+ 2021-07-13 00:24:24,495 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
83
+ 2021-07-13 00:24:26,496 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
84
+ 2021-07-13 00:24:28,496 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
85
+ 2021-07-13 00:24:30,497 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
86
+ 2021-07-13 00:24:32,154 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
87
+ 2021-07-13 00:24:32,498 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
88
+ 2021-07-13 00:24:34,499 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
89
+ 2021-07-13 00:24:35,771 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
90
+ 2021-07-13 00:24:35,771 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
91
+ 2021-07-13 00:24:36,500 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
92
+ 2021-07-13 00:24:38,500 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
93
+ 2021-07-13 00:24:40,501 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
94
+ 2021-07-13 00:24:42,502 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
95
+ 2021-07-13 00:24:44,502 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
96
+ 2021-07-13 00:24:46,503 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
97
+ 2021-07-13 00:24:48,504 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
98
+ 2021-07-13 00:24:50,505 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
99
+ 2021-07-13 00:24:50,909 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
100
+ 2021-07-13 00:24:50,910 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
101
+ 2021-07-13 00:24:52,506 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
102
+ 2021-07-13 00:24:54,506 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
103
+ 2021-07-13 00:24:56,507 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
104
+ 2021-07-13 00:24:58,508 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
105
+ 2021-07-13 00:25:00,509 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
106
+ 2021-07-13 00:25:02,232 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
107
+ 2021-07-13 00:25:02,509 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
108
+ 2021-07-13 00:25:04,510 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
109
+ 2021-07-13 00:25:06,066 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
110
+ 2021-07-13 00:25:06,067 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
111
+ 2021-07-13 00:25:06,511 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
112
+ 2021-07-13 00:25:08,512 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
113
+ 2021-07-13 00:25:10,513 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
114
+ 2021-07-13 00:25:12,513 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
115
+ 2021-07-13 00:25:14,514 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
116
+ 2021-07-13 00:25:16,515 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
117
+ 2021-07-13 00:25:18,516 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
118
+ 2021-07-13 00:25:20,517 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
119
+ 2021-07-13 00:25:21,197 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
120
+ 2021-07-13 00:25:21,198 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
121
+ 2021-07-13 00:25:22,518 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
122
+ 2021-07-13 00:25:24,518 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
123
+ 2021-07-13 00:25:26,519 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
124
+ 2021-07-13 00:25:28,520 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
125
+ 2021-07-13 00:25:30,521 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
126
+ 2021-07-13 00:25:32,322 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
127
+ 2021-07-13 00:25:32,521 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
128
+ 2021-07-13 00:25:34,522 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
129
+ 2021-07-13 00:25:36,329 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
130
+ 2021-07-13 00:25:36,329 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
131
+ 2021-07-13 00:25:36,523 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
132
+ 2021-07-13 00:25:39,524 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
133
+ 2021-07-13 00:25:41,525 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
134
+ 2021-07-13 00:25:43,526 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
135
+ 2021-07-13 00:25:45,526 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
136
+ 2021-07-13 00:25:47,527 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
137
+ 2021-07-13 00:25:49,528 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
138
+ 2021-07-13 00:25:51,529 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
139
+ 2021-07-13 00:25:51,530 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
140
+ 2021-07-13 00:25:51,530 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
141
+ 2021-07-13 00:25:53,530 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
142
+ 2021-07-13 00:25:55,531 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
143
+ 2021-07-13 00:25:57,531 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
144
+ 2021-07-13 00:25:59,532 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
145
+ 2021-07-13 00:26:01,533 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
146
+ 2021-07-13 00:26:02,385 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
147
+ 2021-07-13 00:26:03,534 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
148
+ 2021-07-13 00:26:05,535 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
149
+ 2021-07-13 00:26:06,658 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
150
+ 2021-07-13 00:26:06,659 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
151
+ 2021-07-13 00:26:07,535 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
152
+ 2021-07-13 00:26:09,536 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
153
+ 2021-07-13 00:26:11,537 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
154
+ 2021-07-13 00:26:13,538 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
155
+ 2021-07-13 00:26:15,538 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
156
+ 2021-07-13 00:26:17,539 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
157
+ 2021-07-13 00:26:19,540 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
158
+ 2021-07-13 00:26:21,541 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
159
+ 2021-07-13 00:26:21,788 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
160
+ 2021-07-13 00:26:21,788 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
161
+ 2021-07-13 00:26:23,542 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
162
+ 2021-07-13 00:26:25,542 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
163
+ 2021-07-13 00:26:27,543 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
164
+ 2021-07-13 00:26:29,544 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
165
+ 2021-07-13 00:26:31,545 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
166
+ 2021-07-13 00:26:32,454 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
167
+ 2021-07-13 00:26:33,545 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
168
+ 2021-07-13 00:26:35,546 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
169
+ 2021-07-13 00:26:36,954 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
170
+ 2021-07-13 00:26:36,954 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
171
+ 2021-07-13 00:26:37,547 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
172
+ 2021-07-13 00:26:39,548 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
173
+ 2021-07-13 00:26:41,548 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
174
+ 2021-07-13 00:26:43,549 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
175
+ 2021-07-13 00:26:45,550 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
176
+ 2021-07-13 00:26:47,551 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
177
+ 2021-07-13 00:26:49,552 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
178
+ 2021-07-13 00:26:51,553 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
179
+ 2021-07-13 00:26:52,093 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
180
+ 2021-07-13 00:26:52,094 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
181
+ 2021-07-13 00:26:53,553 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
182
+ 2021-07-13 00:26:55,554 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
183
+ 2021-07-13 00:26:57,555 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
184
+ 2021-07-13 00:26:59,556 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
185
+ 2021-07-13 00:27:01,556 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
186
+ 2021-07-13 00:27:02,524 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
187
+ 2021-07-13 00:27:03,557 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
188
+ 2021-07-13 00:27:05,558 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
189
+ 2021-07-13 00:27:07,225 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
190
+ 2021-07-13 00:27:07,225 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
191
+ 2021-07-13 00:27:07,559 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
192
+ 2021-07-13 00:27:09,559 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
193
+ 2021-07-13 00:27:11,560 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
194
+ 2021-07-13 00:27:13,561 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
195
+ 2021-07-13 00:27:15,562 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
196
+ 2021-07-13 00:27:17,563 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
197
+ 2021-07-13 00:27:19,563 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
198
+ 2021-07-13 00:27:21,564 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
199
+ 2021-07-13 00:27:22,412 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
200
+ 2021-07-13 00:27:22,412 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
201
+ 2021-07-13 00:27:23,565 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
202
+ 2021-07-13 00:27:25,566 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
203
+ 2021-07-13 00:27:27,566 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
204
+ 2021-07-13 00:27:29,567 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
205
+ 2021-07-13 00:27:31,568 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
206
+ 2021-07-13 00:27:32,597 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
207
+ 2021-07-13 00:27:33,569 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
208
+ 2021-07-13 00:27:35,569 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
209
+ 2021-07-13 00:27:37,542 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
210
+ 2021-07-13 00:27:37,542 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
211
+ 2021-07-13 00:27:37,570 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
212
+ 2021-07-13 00:27:39,571 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
213
+ 2021-07-13 00:27:41,572 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
214
+ 2021-07-13 00:27:43,572 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
215
+ 2021-07-13 00:27:45,573 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
216
+ 2021-07-13 00:27:47,574 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
217
+ 2021-07-13 00:27:49,575 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
218
+ 2021-07-13 00:27:51,575 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
219
+ 2021-07-13 00:27:52,672 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
220
+ 2021-07-13 00:27:52,673 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
221
+ 2021-07-13 00:27:53,576 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
222
+ 2021-07-13 00:27:55,577 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
223
+ 2021-07-13 00:27:57,578 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
224
+ 2021-07-13 00:27:59,578 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
225
+ 2021-07-13 00:28:01,579 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
226
+ 2021-07-13 00:28:02,671 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
227
+ 2021-07-13 00:28:03,580 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
228
+ 2021-07-13 00:28:05,581 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
229
+ 2021-07-13 00:28:07,582 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
230
+ 2021-07-13 00:28:07,868 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
231
+ 2021-07-13 00:28:07,869 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
232
+ 2021-07-13 00:28:10,583 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
233
+ 2021-07-13 00:28:12,583 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
234
+ 2021-07-13 00:28:14,584 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
235
+ 2021-07-13 00:28:16,585 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
236
+ 2021-07-13 00:28:18,586 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
237
+ 2021-07-13 00:28:20,587 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
238
+ 2021-07-13 00:28:22,588 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
239
+ 2021-07-13 00:28:22,998 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
240
+ 2021-07-13 00:28:22,998 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
241
+ 2021-07-13 00:28:24,588 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
242
+ 2021-07-13 00:28:26,589 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
243
+ 2021-07-13 00:28:28,590 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
244
+ 2021-07-13 00:28:30,591 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
245
+ 2021-07-13 00:28:32,592 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
246
+ 2021-07-13 00:28:32,746 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
247
+ 2021-07-13 00:28:34,593 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
248
+ 2021-07-13 00:28:38,128 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
249
+ 2021-07-13 00:28:38,128 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
250
+ 2021-07-13 00:28:52,048 DEBUG SenderThread:320742 [sender.py:send():179] send: history
251
+ 2021-07-13 00:28:52,049 DEBUG SenderThread:320742 [sender.py:send():179] send: summary
252
+ 2021-07-13 00:28:52,049 INFO SenderThread:320742 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
253
+ 2021-07-13 00:28:52,600 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/wandb-summary.json
254
+ 2021-07-13 00:28:53,447 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
255
+ 2021-07-13 00:28:53,447 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
256
+ 2021-07-13 00:28:54,601 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
257
+ 2021-07-13 00:28:56,601 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
258
+ 2021-07-13 00:28:58,602 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
259
+ 2021-07-13 00:29:00,603 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
260
+ 2021-07-13 00:29:02,604 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
261
+ 2021-07-13 00:29:02,822 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
262
+ 2021-07-13 00:29:04,605 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
263
+ 2021-07-13 00:29:06,606 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
264
+ 2021-07-13 00:29:08,607 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
265
+ 2021-07-13 00:29:08,643 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
266
+ 2021-07-13 00:29:08,643 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
267
+ 2021-07-13 00:29:10,607 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
268
+ 2021-07-13 00:29:12,608 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
269
+ 2021-07-13 00:29:14,609 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
270
+ 2021-07-13 00:29:16,610 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
271
+ 2021-07-13 00:29:18,611 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
272
+ 2021-07-13 00:29:20,612 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
273
+ 2021-07-13 00:29:22,612 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
274
+ 2021-07-13 00:29:23,774 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
275
+ 2021-07-13 00:29:23,774 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
276
+ 2021-07-13 00:29:24,613 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
277
+ 2021-07-13 00:29:26,614 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
278
+ 2021-07-13 00:29:28,615 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
279
+ 2021-07-13 00:29:30,616 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
280
+ 2021-07-13 00:29:32,617 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
281
+ 2021-07-13 00:29:32,899 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
282
+ 2021-07-13 00:29:34,618 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
283
+ 2021-07-13 00:29:36,618 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
284
+ 2021-07-13 00:29:38,619 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
285
+ 2021-07-13 00:29:38,915 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
286
+ 2021-07-13 00:29:38,916 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
287
+ 2021-07-13 00:29:40,620 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
288
+ 2021-07-13 00:29:42,621 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
289
+ 2021-07-13 00:29:44,621 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
290
+ 2021-07-13 00:29:46,622 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
291
+ 2021-07-13 00:29:48,623 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
292
+ 2021-07-13 00:29:50,624 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
293
+ 2021-07-13 00:29:52,625 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
294
+ 2021-07-13 00:29:54,045 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
295
+ 2021-07-13 00:29:54,046 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
296
+ 2021-07-13 00:29:54,626 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
297
+ 2021-07-13 00:29:56,627 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
298
+ 2021-07-13 00:29:58,628 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
299
+ 2021-07-13 00:30:00,629 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
300
+ 2021-07-13 00:30:02,630 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
301
+ 2021-07-13 00:30:02,986 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
302
+ 2021-07-13 00:30:04,631 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
303
+ 2021-07-13 00:30:06,632 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
304
+ 2021-07-13 00:30:08,632 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
305
+ 2021-07-13 00:30:09,182 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
306
+ 2021-07-13 00:30:09,183 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
307
+ 2021-07-13 00:30:10,633 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
308
+ 2021-07-13 00:30:12,634 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
309
+ 2021-07-13 00:30:14,635 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
310
+ 2021-07-13 00:30:16,636 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
311
+ 2021-07-13 00:30:18,637 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
312
+ 2021-07-13 00:30:20,638 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
313
+ 2021-07-13 00:30:22,638 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
314
+ 2021-07-13 00:30:24,346 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
315
+ 2021-07-13 00:30:24,347 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
316
+ 2021-07-13 00:30:24,639 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
317
+ 2021-07-13 00:30:26,640 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
318
+ 2021-07-13 00:30:28,641 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
319
+ 2021-07-13 00:30:30,642 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
320
+ 2021-07-13 00:30:32,642 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
321
+ 2021-07-13 00:30:33,062 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
322
+ 2021-07-13 00:30:34,643 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
323
+ 2021-07-13 00:30:36,644 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
324
+ 2021-07-13 00:30:38,645 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
325
+ 2021-07-13 00:30:39,476 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
326
+ 2021-07-13 00:30:39,477 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
327
+ 2021-07-13 00:30:40,646 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
328
+ 2021-07-13 00:30:42,646 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
329
+ 2021-07-13 00:30:44,647 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
330
+ 2021-07-13 00:30:46,648 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
331
+ 2021-07-13 00:30:48,649 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
332
+ 2021-07-13 00:30:50,650 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
333
+ 2021-07-13 00:30:52,651 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
334
+ 2021-07-13 00:30:54,608 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
335
+ 2021-07-13 00:30:54,608 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
336
+ 2021-07-13 00:30:54,652 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
337
+ 2021-07-13 00:30:56,652 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
338
+ 2021-07-13 00:30:58,653 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
339
+ 2021-07-13 00:31:00,654 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
340
+ 2021-07-13 00:31:02,655 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
341
+ 2021-07-13 00:31:03,139 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
342
+ 2021-07-13 00:31:04,656 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
343
+ 2021-07-13 00:31:06,656 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
344
+ 2021-07-13 00:31:08,657 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
345
+ 2021-07-13 00:31:09,738 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
346
+ 2021-07-13 00:31:09,738 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
347
+ 2021-07-13 00:31:11,658 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
348
+ 2021-07-13 00:31:13,659 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
349
+ 2021-07-13 00:31:15,660 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
350
+ 2021-07-13 00:31:17,661 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
351
+ 2021-07-13 00:31:19,662 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
352
+ 2021-07-13 00:31:21,663 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
353
+ 2021-07-13 00:31:23,663 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
354
+ 2021-07-13 00:31:24,869 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
355
+ 2021-07-13 00:31:24,869 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
356
+ 2021-07-13 00:31:25,664 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
357
+ 2021-07-13 00:31:27,665 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
358
+ 2021-07-13 00:31:29,666 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
359
+ 2021-07-13 00:31:31,667 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
360
+ 2021-07-13 00:31:33,216 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
361
+ 2021-07-13 00:31:33,668 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
362
+ 2021-07-13 00:31:35,669 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
363
+ 2021-07-13 00:31:37,670 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
364
+ 2021-07-13 00:31:39,671 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
365
+ 2021-07-13 00:31:40,001 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
366
+ 2021-07-13 00:31:40,001 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
367
+ 2021-07-13 00:31:41,672 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
368
+ 2021-07-13 00:31:43,673 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
369
+ 2021-07-13 00:31:45,674 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
370
+ 2021-07-13 00:31:47,674 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
371
+ 2021-07-13 00:31:49,675 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
372
+ 2021-07-13 00:31:51,676 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
373
+ 2021-07-13 00:31:53,677 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
374
+ 2021-07-13 00:31:55,133 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
375
+ 2021-07-13 00:31:55,134 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
376
+ 2021-07-13 00:31:55,678 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
377
+ 2021-07-13 00:31:57,679 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
378
+ 2021-07-13 00:31:59,680 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
379
+ 2021-07-13 00:32:01,681 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
380
+ 2021-07-13 00:32:03,294 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
381
+ 2021-07-13 00:32:03,682 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
382
+ 2021-07-13 00:32:05,683 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
383
+ 2021-07-13 00:32:07,684 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
384
+ 2021-07-13 00:32:09,685 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
385
+ 2021-07-13 00:32:10,266 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
386
+ 2021-07-13 00:32:10,266 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
387
+ 2021-07-13 00:32:11,686 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
388
+ 2021-07-13 00:32:13,687 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
389
+ 2021-07-13 00:32:15,688 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
390
+ 2021-07-13 00:32:17,689 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
391
+ 2021-07-13 00:32:19,690 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
392
+ 2021-07-13 00:32:21,691 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
393
+ 2021-07-13 00:32:23,691 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
394
+ 2021-07-13 00:32:25,396 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
395
+ 2021-07-13 00:32:25,396 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
396
+ 2021-07-13 00:32:25,692 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
397
+ 2021-07-13 00:32:27,693 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
398
+ 2021-07-13 00:32:29,694 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
399
+ 2021-07-13 00:32:31,695 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
400
+ 2021-07-13 00:32:33,373 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
401
+ 2021-07-13 00:32:33,696 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
402
+ 2021-07-13 00:32:35,697 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
403
+ 2021-07-13 00:32:37,698 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
404
+ 2021-07-13 00:32:39,699 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
405
+ 2021-07-13 00:32:40,527 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
406
+ 2021-07-13 00:32:40,527 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
407
+ 2021-07-13 00:32:41,700 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
408
+ 2021-07-13 00:32:43,701 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
409
+ 2021-07-13 00:32:45,702 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
410
+ 2021-07-13 00:32:47,703 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
411
+ 2021-07-13 00:32:49,704 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
412
+ 2021-07-13 00:32:51,704 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
413
+ 2021-07-13 00:32:53,705 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
414
+ 2021-07-13 00:32:55,686 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
415
+ 2021-07-13 00:32:55,686 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
416
+ 2021-07-13 00:32:55,706 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
417
+ 2021-07-13 00:32:57,707 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
418
+ 2021-07-13 00:32:59,708 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
419
+ 2021-07-13 00:33:01,709 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
420
+ 2021-07-13 00:33:03,450 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
421
+ 2021-07-13 00:33:03,710 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
422
+ 2021-07-13 00:33:05,711 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
423
+ 2021-07-13 00:33:07,712 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
424
+ 2021-07-13 00:33:09,713 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
425
+ 2021-07-13 00:33:10,817 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
426
+ 2021-07-13 00:33:10,817 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
427
+ 2021-07-13 00:33:11,714 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
428
+ 2021-07-13 00:33:13,715 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
429
+ 2021-07-13 00:33:15,716 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
430
+ 2021-07-13 00:33:17,717 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
431
+ 2021-07-13 00:33:19,718 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
432
+ 2021-07-13 00:33:21,719 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
433
+ 2021-07-13 00:33:25,951 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
434
+ 2021-07-13 00:33:25,951 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
435
+ 2021-07-13 00:33:33,528 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
436
+ 2021-07-13 00:33:40,391 DEBUG SenderThread:320742 [sender.py:send():179] send: history
437
+ 2021-07-13 00:33:40,392 DEBUG SenderThread:320742 [sender.py:send():179] send: summary
438
+ 2021-07-13 00:33:40,393 INFO SenderThread:320742 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
439
+ 2021-07-13 00:33:40,727 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/wandb-summary.json
440
+ 2021-07-13 00:33:41,082 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
441
+ 2021-07-13 00:33:41,083 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
442
+ 2021-07-13 00:33:41,727 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
443
+ 2021-07-13 00:33:43,728 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
444
+ 2021-07-13 00:33:45,729 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
445
+ 2021-07-13 00:33:47,730 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
446
+ 2021-07-13 00:33:49,731 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
447
+ 2021-07-13 00:33:51,732 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
448
+ 2021-07-13 00:33:53,732 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
449
+ 2021-07-13 00:33:55,733 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
450
+ 2021-07-13 00:33:56,269 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
451
+ 2021-07-13 00:33:56,269 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
452
+ 2021-07-13 00:33:57,734 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
453
+ 2021-07-13 00:33:59,735 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
454
+ 2021-07-13 00:34:01,736 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
455
+ 2021-07-13 00:34:03,606 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
456
+ 2021-07-13 00:34:03,737 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
457
+ 2021-07-13 00:34:05,738 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
458
+ 2021-07-13 00:34:07,739 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
459
+ 2021-07-13 00:34:09,740 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
460
+ 2021-07-13 00:34:11,403 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
461
+ 2021-07-13 00:34:11,403 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
462
+ 2021-07-13 00:34:12,741 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
463
+ 2021-07-13 00:34:14,742 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
464
+ 2021-07-13 00:34:16,743 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
465
+ 2021-07-13 00:34:18,744 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
466
+ 2021-07-13 00:34:20,745 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
467
+ 2021-07-13 00:34:22,746 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
468
+ 2021-07-13 00:34:24,747 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
469
+ 2021-07-13 00:34:26,534 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
470
+ 2021-07-13 00:34:26,535 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
471
+ 2021-07-13 00:34:26,747 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
472
+ 2021-07-13 00:34:28,748 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
473
+ 2021-07-13 00:34:30,749 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
474
+ 2021-07-13 00:34:32,750 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
475
+ 2021-07-13 00:34:33,683 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
476
+ 2021-07-13 00:34:34,751 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
477
+ 2021-07-13 00:34:36,752 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
478
+ 2021-07-13 00:34:38,753 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
479
+ 2021-07-13 00:34:40,754 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
480
+ 2021-07-13 00:34:41,673 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
481
+ 2021-07-13 00:34:41,674 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
482
+ 2021-07-13 00:34:42,755 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
483
+ 2021-07-13 00:34:44,756 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
484
+ 2021-07-13 00:34:46,757 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
485
+ 2021-07-13 00:34:48,758 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
486
+ 2021-07-13 00:34:50,759 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
487
+ 2021-07-13 00:34:52,759 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
488
+ 2021-07-13 00:34:54,760 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
489
+ 2021-07-13 00:34:56,761 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
490
+ 2021-07-13 00:34:56,805 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
491
+ 2021-07-13 00:34:56,805 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
492
+ 2021-07-13 00:34:58,762 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
493
+ 2021-07-13 00:35:00,763 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
494
+ 2021-07-13 00:35:02,764 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
495
+ 2021-07-13 00:35:03,761 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
496
+ 2021-07-13 00:35:04,765 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
497
+ 2021-07-13 00:35:06,766 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
498
+ 2021-07-13 00:35:08,767 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
499
+ 2021-07-13 00:35:10,768 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
500
+ 2021-07-13 00:35:11,934 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
501
+ 2021-07-13 00:35:11,935 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
502
+ 2021-07-13 00:35:12,769 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
503
+ 2021-07-13 00:35:14,770 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
504
+ 2021-07-13 00:35:16,771 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
505
+ 2021-07-13 00:35:18,771 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
506
+ 2021-07-13 00:35:20,772 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
507
+ 2021-07-13 00:35:22,773 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
508
+ 2021-07-13 00:35:24,774 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
509
+ 2021-07-13 00:35:26,775 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
510
+ 2021-07-13 00:35:27,118 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
511
+ 2021-07-13 00:35:27,118 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
512
+ 2021-07-13 00:35:28,776 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
513
+ 2021-07-13 00:35:30,777 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
514
+ 2021-07-13 00:35:32,778 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
515
+ 2021-07-13 00:35:33,840 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
516
+ 2021-07-13 00:35:34,779 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
517
+ 2021-07-13 00:35:36,780 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
518
+ 2021-07-13 00:35:38,781 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
519
+ 2021-07-13 00:35:40,781 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
520
+ 2021-07-13 00:35:42,250 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
521
+ 2021-07-13 00:35:42,250 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
522
+ 2021-07-13 00:35:42,782 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
523
+ 2021-07-13 00:35:44,783 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
524
+ 2021-07-13 00:35:46,784 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
525
+ 2021-07-13 00:35:48,785 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
526
+ 2021-07-13 00:35:50,786 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
527
+ 2021-07-13 00:35:52,787 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
528
+ 2021-07-13 00:35:54,788 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
529
+ 2021-07-13 00:35:56,789 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
530
+ 2021-07-13 00:35:57,380 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
531
+ 2021-07-13 00:35:57,380 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
532
+ 2021-07-13 00:35:58,790 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
533
+ 2021-07-13 00:36:00,791 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
534
+ 2021-07-13 00:36:02,792 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
535
+ 2021-07-13 00:36:03,916 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
536
+ 2021-07-13 00:36:04,792 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
537
+ 2021-07-13 00:36:06,793 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
538
+ 2021-07-13 00:36:08,794 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
539
+ 2021-07-13 00:36:11,795 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
540
+ 2021-07-13 00:36:12,574 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
541
+ 2021-07-13 00:36:12,574 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
542
+ 2021-07-13 00:36:13,796 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
543
+ 2021-07-13 00:36:15,797 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
544
+ 2021-07-13 00:36:17,798 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
545
+ 2021-07-13 00:36:19,799 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
546
+ 2021-07-13 00:36:21,799 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
547
+ 2021-07-13 00:36:23,800 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
548
+ 2021-07-13 00:36:25,801 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
549
+ 2021-07-13 00:36:27,703 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
550
+ 2021-07-13 00:36:27,703 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
551
+ 2021-07-13 00:36:27,802 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
552
+ 2021-07-13 00:36:29,803 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
553
+ 2021-07-13 00:36:31,804 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
554
+ 2021-07-13 00:36:33,805 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
555
+ 2021-07-13 00:36:33,992 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
556
+ 2021-07-13 00:36:35,806 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
557
+ 2021-07-13 00:36:37,806 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
558
+ 2021-07-13 00:36:39,807 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
559
+ 2021-07-13 00:36:41,808 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
560
+ 2021-07-13 00:36:42,837 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
561
+ 2021-07-13 00:36:42,837 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
562
+ 2021-07-13 00:36:43,809 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
563
+ 2021-07-13 00:36:45,809 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
564
+ 2021-07-13 00:36:47,810 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
565
+ 2021-07-13 00:36:49,811 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
566
+ 2021-07-13 00:36:51,812 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
567
+ 2021-07-13 00:36:53,813 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
568
+ 2021-07-13 00:36:55,813 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
569
+ 2021-07-13 00:36:57,814 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
570
+ 2021-07-13 00:36:58,033 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
571
+ 2021-07-13 00:36:58,034 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
572
+ 2021-07-13 00:36:59,815 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
573
+ 2021-07-13 00:37:01,816 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
574
+ 2021-07-13 00:37:03,817 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
575
+ 2021-07-13 00:37:04,067 DEBUG SenderThread:320742 [sender.py:send():179] send: stats
576
+ 2021-07-13 00:37:05,818 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
577
+ 2021-07-13 00:37:07,819 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
578
+ 2021-07-13 00:37:09,819 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
579
+ 2021-07-13 00:37:11,820 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
580
+ 2021-07-13 00:37:13,164 DEBUG HandlerThread:320742 [handler.py:handle_request():124] handle_request: stop_status
581
+ 2021-07-13 00:37:13,164 DEBUG SenderThread:320742 [sender.py:send_request():193] send_request: stop_status
582
+ 2021-07-13 00:37:13,821 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
583
+ 2021-07-13 00:37:15,822 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
584
+ 2021-07-13 00:37:17,823 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
585
+ 2021-07-13 00:37:19,824 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
586
+ 2021-07-13 00:37:21,824 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
587
+ 2021-07-13 00:37:23,825 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
588
+ 2021-07-13 00:37:25,826 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
589
+ 2021-07-13 00:37:27,492 WARNING MainThread:320742 [internal.py:wandb_internal():147] Internal process interrupt: 1
590
+ 2021-07-13 00:37:27,730 WARNING MainThread:320742 [internal.py:wandb_internal():147] Internal process interrupt: 2
591
+ 2021-07-13 00:37:27,731 ERROR MainThread:320742 [internal.py:wandb_internal():150] Internal process interrupted.
592
+ 2021-07-13 00:37:27,827 INFO Thread-8 :320742 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log
593
+ 2021-07-13 00:37:28,330 INFO SenderThread:320742 [sender.py:finish():945] shutting down sender
594
+ 2021-07-13 00:37:28,330 INFO SenderThread:320742 [dir_watcher.py:finish():282] shutting down directory watcher
595
+ 2021-07-13 00:37:28,330 INFO WriterThread:320742 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/run-11bfig2u.wandb
596
+ 2021-07-13 00:37:28,331 INFO HandlerThread:320742 [handler.py:finish():638] shutting down handler
597
+ 2021-07-13 00:37:28,828 INFO SenderThread:320742 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files
598
+ 2021-07-13 00:37:28,829 INFO SenderThread:320742 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/requirements.txt requirements.txt
599
+ 2021-07-13 00:37:28,829 INFO SenderThread:320742 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/output.log output.log
600
+ 2021-07-13 00:37:28,829 INFO SenderThread:320742 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/wandb-metadata.json wandb-metadata.json
601
+ 2021-07-13 00:37:28,829 INFO SenderThread:320742 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/config.yaml config.yaml
602
+ 2021-07-13 00:37:28,829 INFO SenderThread:320742 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/files/wandb-summary.json wandb-summary.json
603
+ 2021-07-13 00:37:28,830 INFO SenderThread:320742 [file_pusher.py:finish():177] shutting down file pusher
604
+ 2021-07-13 00:37:28,830 INFO SenderThread:320742 [file_pusher.py:join():182] waiting for file pusher
605
+ 2021-07-13 00:37:29,011 INFO MainThread:320742 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210713_002031-11bfig2u/logs/debug.log ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-13 00:20:31,458 INFO MainThread:319169 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-13 00:20:31,458 INFO MainThread:319169 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-13 00:20:31,458 INFO MainThread:319169 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/logs/debug.log
4
+ 2021-07-13 00:20:31,458 INFO MainThread:319169 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210713_002031-11bfig2u/logs/debug-internal.log
5
+ 2021-07-13 00:20:31,459 INFO MainThread:319169 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-13 00:20:31,459 INFO MainThread:319169 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-13 00:20:31,459 INFO MainThread:319169 [wandb_init.py:init():419] starting backend
9
+ 2021-07-13 00:20:31,459 INFO MainThread:319169 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-13 00:20:31,517 INFO MainThread:319169 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-13 00:20:31,574 INFO MainThread:319169 [backend.py:ensure_launched():139] started backend process with pid: 320742
12
+ 2021-07-13 00:20:31,576 INFO MainThread:319169 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-13 00:20:31,579 INFO MainThread:319169 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-13 00:20:31,580 INFO MainThread:319169 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-13 00:20:32,227 INFO MainThread:319169 [wandb_init.py:init():496] got version response
16
+ 2021-07-13 00:20:32,227 INFO MainThread:319169 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
17
+ 2021-07-13 00:20:32,409 INFO MainThread:319169 [wandb_init.py:init():529] starting run threads in backend
18
+ 2021-07-13 00:20:33,559 INFO MainThread:319169 [wandb_run.py:_console_start():1623] atexit reg
19
+ 2021-07-13 00:20:33,559 INFO MainThread:319169 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
20
+ 2021-07-13 00:20:33,560 INFO MainThread:319169 [wandb_run.py:_redirect():1502] Redirecting console.
21
+ 2021-07-13 00:20:33,562 INFO MainThread:319169 [wandb_run.py:_redirect():1558] Redirects installed.
22
+ 2021-07-13 00:20:33,562 INFO MainThread:319169 [wandb_init.py:init():554] run started, returning control to user process
23
+ 2021-07-13 00:20:33,568 INFO MainThread:319169 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul13_00-19-44_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 20000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 92768, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': True, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
+ 2021-07-13 00:20:33,570 INFO MainThread:319169 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
25
+ 2021-07-13 00:20:33,571 INFO MainThread:319169 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 64, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False}
26
+ 2021-07-13 00:37:27,852 INFO MainThread:319169 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
27
+ 2021-07-13 00:37:27,853 INFO MainThread:319169 [wandb_run.py:_restore():1565] restore
wandb/run-20210713_002031-11bfig2u/run-11bfig2u.wandb ADDED
Binary file (53.3 kB). View file