dat commited on
Commit
7398222
1 Parent(s): b439f0a

add pt tokenizer

Browse files
Load data & train tokenizer.ipynb CHANGED
@@ -44,7 +44,7 @@
44
  },
45
  {
46
  "cell_type": "code",
47
- "execution_count": 74,
48
  "id": "9241a429",
49
  "metadata": {},
50
  "outputs": [],
@@ -54,25 +54,17 @@
54
  },
55
  {
56
  "cell_type": "code",
57
- "execution_count": 75,
58
  "id": "348a4dd4",
59
  "metadata": {},
60
- "outputs": [
61
- {
62
- "name": "stderr",
63
- "output_type": "stream",
64
- "text": [
65
- "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
66
- ]
67
- }
68
- ],
69
  "source": [
70
  "tokenizer = AutoTokenizer.from_pretrained(\"./\")"
71
  ]
72
  },
73
  {
74
  "cell_type": "code",
75
- "execution_count": 81,
76
  "id": "595f318e",
77
  "metadata": {},
78
  "outputs": [
@@ -82,7 +74,7 @@
82
  "36450"
83
  ]
84
  },
85
- "execution_count": 81,
86
  "metadata": {},
87
  "output_type": "execute_result"
88
  }
@@ -91,6 +83,58 @@
91
  "tokenizer.vocab_size"
92
  ]
93
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  {
95
  "cell_type": "code",
96
  "execution_count": 3,
 
44
  },
45
  {
46
  "cell_type": "code",
47
+ "execution_count": 1,
48
  "id": "9241a429",
49
  "metadata": {},
50
  "outputs": [],
 
54
  },
55
  {
56
  "cell_type": "code",
57
+ "execution_count": 2,
58
  "id": "348a4dd4",
59
  "metadata": {},
60
+ "outputs": [],
 
 
 
 
 
 
 
 
61
  "source": [
62
  "tokenizer = AutoTokenizer.from_pretrained(\"./\")"
63
  ]
64
  },
65
  {
66
  "cell_type": "code",
67
+ "execution_count": 4,
68
  "id": "595f318e",
69
  "metadata": {},
70
  "outputs": [
 
74
  "36450"
75
  ]
76
  },
77
+ "execution_count": 4,
78
  "metadata": {},
79
  "output_type": "execute_result"
80
  }
 
83
  "tokenizer.vocab_size"
84
  ]
85
  },
86
+ {
87
+ "cell_type": "code",
88
+ "execution_count": 45,
89
+ "id": "e12a921a",
90
+ "metadata": {},
91
+ "outputs": [
92
+ {
93
+ "data": {
94
+ "text/plain": [
95
+ "{'input_ids': [4, 3620], 'attention_mask': [1, 1]}"
96
+ ]
97
+ },
98
+ "execution_count": 45,
99
+ "metadata": {},
100
+ "output_type": "execute_result"
101
+ }
102
+ ],
103
+ "source": [
104
+ "tokenizer(\"<mask> word\")"
105
+ ]
106
+ },
107
+ {
108
+ "cell_type": "code",
109
+ "execution_count": null,
110
+ "id": "441cd5af",
111
+ "metadata": {},
112
+ "outputs": [],
113
+ "source": [
114
+ ".mask_token = "
115
+ ]
116
+ },
117
+ {
118
+ "cell_type": "code",
119
+ "execution_count": 46,
120
+ "id": "cbb3b0bc",
121
+ "metadata": {},
122
+ "outputs": [
123
+ {
124
+ "data": {
125
+ "text/plain": [
126
+ "{'input_ids': [4, 3620], 'attention_mask': [1, 1]}"
127
+ ]
128
+ },
129
+ "execution_count": 46,
130
+ "metadata": {},
131
+ "output_type": "execute_result"
132
+ }
133
+ ],
134
+ "source": [
135
+ "tokenizer(\"<mask> word\")"
136
+ ]
137
+ },
138
  {
139
  "cell_type": "code",
140
  "execution_count": 3,
create_pt_tokenizer.py CHANGED
@@ -1,3 +1,4 @@
1
- from transformers import AutoTokenizer
2
  tokenizer = AutoTokenizer.from_pretrained("./")
 
3
  tokenizer.save_pretrained("./")
 
1
+ from transformers import AutoTokenizer, AddedToken
2
  tokenizer = AutoTokenizer.from_pretrained("./")
3
+ tokenizer.mask_token = AddedToken("<mask>", lstrip=True)
4
  tokenizer.save_pretrained("./")
special_tokens_map.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}
 
1
+ {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}
wandb/run-20210715_185845-dq8uirtg/files/output.log CHANGED
@@ -3331,3 +3331,282 @@ Training...: 28805it [3:24:58, 1.09it/s]█████████████
3331
 
3332
 
3333
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3331
 
3332
 
3333
 
3334
+
3335
+ Training...: 28855it [3:25:18, 1.09it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3336
+
3337
+
3338
+
3339
+
3340
+
3341
+ Training...: 28905it [3:25:38, 1.09it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3342
+
3343
+
3344
+
3345
+
3346
+ Training...: 28948it [3:25:45, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3347
+
3348
+
3349
+
3350
+
3351
+
3352
+ Training...: 28998it [3:26:05, 2.67it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3353
+
3354
+
3355
+
3356
+
3357
+
3358
+ Training...: 29048it [3:26:25, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3359
+
3360
+
3361
+
3362
+
3363
+
3364
+ Training...: 29098it [3:26:45, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3365
+
3366
+
3367
+
3368
+
3369
+
3370
+ Training...: 29148it [3:27:05, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3371
+
3372
+
3373
+
3374
+
3375
+
3376
+
3377
+ Training...: 29209it [3:27:39, 2.23it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3378
+
3379
+
3380
+
3381
+
3382
+ Training...: 29261it [3:27:59, 3.04it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3383
+
3384
+
3385
+
3386
+
3387
+ Training...: 29298it [3:28:05, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3388
+
3389
+
3390
+
3391
+
3392
+
3393
+ Training...: 29348it [3:28:25, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3394
+
3395
+
3396
+
3397
+
3398
+
3399
+ Training...: 29398it [3:28:45, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3400
+
3401
+
3402
+
3403
+
3404
+
3405
+ Training...: 29448it [3:29:05, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3406
+
3407
+
3408
+
3409
+
3410
+
3411
+ Training...: 29498it [3:29:25, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3412
+
3413
+
3414
+
3415
+
3416
+
3417
+ Training...: 29563it [3:30:00, 4.11it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3418
+
3419
+
3420
+
3421
+
3422
+
3423
+ Training...: 29605it [3:30:19, 1.08it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3424
+
3425
+
3426
+
3427
+
3428
+
3429
+ Training...: 29657it [3:30:39, 1.59it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3430
+
3431
+
3432
+
3433
+
3434
+ Training...: 29698it [3:30:46, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3435
+
3436
+
3437
+
3438
+
3439
+
3440
+ Training...: 29748it [3:31:06, 2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3441
+
3442
+
3443
+
3444
+
3445
+
3446
+ Training...: 29798it [3:31:26, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3447
+
3448
+
3449
+
3450
+
3451
+ Training...: 29848it [3:31:46, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3452
+
3453
+
3454
+
3455
+
3456
+ Training...: 29898it [3:32:06, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3457
+
3458
+
3459
+
3460
+
3461
+ Training...: 29948it [3:32:26, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3462
+
3463
+
3464
+
3465
+
3466
+ Training...: 29998it [3:32:46, 2.74it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3467
+
3468
+
3469
+
3470
+
3471
+
3472
+ Training...: 30048it [3:33:06, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3473
+
3474
+
3475
+
3476
+
3477
+
3478
+ Training...: 30098it [3:33:26, 2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3479
+
3480
+
3481
+
3482
+
3483
+
3484
+ Training...: 30148it [3:33:46, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3485
+
3486
+
3487
+
3488
+
3489
+
3490
+ Training...: 30198it [3:34:06, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3491
+
3492
+
3493
+
3494
+
3495
+ Training...: 30248it [3:34:26, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3496
+
3497
+
3498
+
3499
+
3500
+ Training...: 30298it [3:34:46, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3501
+
3502
+
3503
+
3504
+
3505
+ Training...: 30348it [3:35:06, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3506
+
3507
+
3508
+
3509
+
3510
+
3511
+ Training...: 30398it [3:35:26, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3512
+
3513
+
3514
+
3515
+
3516
+
3517
+ Training...: 30448it [3:35:46, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3518
+
3519
+
3520
+
3521
+
3522
+
3523
+ Training...: 30498it [3:36:06, 2.66it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3524
+
3525
+
3526
+
3527
+
3528
+
3529
+ Training...: 30548it [3:36:26, 2.67it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3530
+
3531
+
3532
+
3533
+
3534
+
3535
+ Training...: 30598it [3:36:46, 2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3536
+
3537
+
3538
+
3539
+
3540
+
3541
+ Training...: 30648it [3:37:07, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3542
+
3543
+
3544
+
3545
+
3546
+ Training...: 30698it [3:37:27, 2.72it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3547
+
3548
+
3549
+
3550
+
3551
+ Training...: 30748it [3:37:47, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3552
+
3553
+
3554
+
3555
+
3556
+
3557
+ Training...: 30798it [3:38:07, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3558
+
3559
+
3560
+
3561
+
3562
+
3563
+ Training...: 30848it [3:38:27, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3564
+
3565
+
3566
+
3567
+
3568
+
3569
+ Training...: 30898it [3:38:47, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3570
+
3571
+
3572
+
3573
+
3574
+
3575
+ Training...: 30948it [3:39:07, 2.73it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3576
+
3577
+
3578
+
3579
+
3580
+
3581
+ Training...: 30998it [3:39:27, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3582
+
3583
+
3584
+
3585
+
3586
+
3587
+ Training...: 31048it [3:39:47, 2.71it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3588
+
3589
+
3590
+
3591
+
3592
+ Training...: 31098it [3:40:07, 2.68it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3593
+
3594
+
3595
+
3596
+
3597
+ Training...: 31148it [3:40:27, 2.70it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3598
+
3599
+
3600
+
3601
+
3602
+ Training...: 31198it [3:40:47, 2.69it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3603
+
3604
+
3605
+
3606
+
3607
+ Training...: 31248it [3:41:07, 2.74it/s]██████████████████████████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
3608
+
3609
+
3610
+
3611
+
3612
+
wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"training_step": 128800, "learning_rate": 2.4875864255591296e-05, "train_loss": 2.341568946838379, "_runtime": 12642, "_timestamp": 1626388167, "_step": 580, "eval_step": 126000, "eval_accuracy": 0.5787546038627625, "eval_loss": 2.237921953201294}
 
1
+ {"training_step": 131250, "learning_rate": 2.4770188247202896e-05, "train_loss": 2.2838871479034424, "_runtime": 13625, "_timestamp": 1626389150, "_step": 629, "eval_step": 126000, "eval_accuracy": 0.5787546038627625, "eval_loss": 2.237921953201294}
wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log CHANGED
@@ -7463,3 +7463,614 @@
7463
  2021-07-15 22:29:36,104 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7464
  2021-07-15 22:29:36,105 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7465
  2021-07-15 22:29:37,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7463
  2021-07-15 22:29:36,104 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7464
  2021-07-15 22:29:36,105 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7465
  2021-07-15 22:29:37,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7466
+ 2021-07-15 22:29:48,034 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7467
+ 2021-07-15 22:29:48,035 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7468
+ 2021-07-15 22:29:48,035 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7469
+ 2021-07-15 22:29:48,825 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7470
+ 2021-07-15 22:29:48,885 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7471
+ 2021-07-15 22:29:49,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7472
+ 2021-07-15 22:29:50,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7473
+ 2021-07-15 22:29:51,235 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7474
+ 2021-07-15 22:29:51,235 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7475
+ 2021-07-15 22:29:51,887 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7476
+ 2021-07-15 22:29:53,888 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7477
+ 2021-07-15 22:29:55,889 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7478
+ 2021-07-15 22:29:57,890 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7479
+ 2021-07-15 22:30:06,376 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7480
+ 2021-07-15 22:30:06,377 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7481
+ 2021-07-15 22:30:08,103 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7482
+ 2021-07-15 22:30:08,103 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7483
+ 2021-07-15 22:30:08,104 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7484
+ 2021-07-15 22:30:08,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7485
+ 2021-07-15 22:30:09,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7486
+ 2021-07-15 22:30:10,895 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7487
+ 2021-07-15 22:30:11,895 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7488
+ 2021-07-15 22:30:13,896 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7489
+ 2021-07-15 22:30:15,897 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7490
+ 2021-07-15 22:30:17,898 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7491
+ 2021-07-15 22:30:18,905 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7492
+ 2021-07-15 22:30:21,509 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7493
+ 2021-07-15 22:30:21,510 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7494
+ 2021-07-15 22:30:28,142 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7495
+ 2021-07-15 22:30:28,142 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7496
+ 2021-07-15 22:30:28,143 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7497
+ 2021-07-15 22:30:28,902 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7498
+ 2021-07-15 22:30:28,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7499
+ 2021-07-15 22:30:29,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7500
+ 2021-07-15 22:30:31,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7501
+ 2021-07-15 22:30:33,904 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7502
+ 2021-07-15 22:30:35,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7503
+ 2021-07-15 22:30:36,640 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7504
+ 2021-07-15 22:30:36,640 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7505
+ 2021-07-15 22:30:37,906 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7506
+ 2021-07-15 22:30:48,194 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7507
+ 2021-07-15 22:30:48,195 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7508
+ 2021-07-15 22:30:48,195 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7509
+ 2021-07-15 22:30:48,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7510
+ 2021-07-15 22:30:48,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7511
+ 2021-07-15 22:30:48,986 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7512
+ 2021-07-15 22:30:49,911 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7513
+ 2021-07-15 22:30:51,772 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7514
+ 2021-07-15 22:30:51,773 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7515
+ 2021-07-15 22:30:51,911 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7516
+ 2021-07-15 22:30:53,912 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7517
+ 2021-07-15 22:30:55,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7518
+ 2021-07-15 22:30:57,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7519
+ 2021-07-15 22:31:06,907 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7520
+ 2021-07-15 22:31:06,908 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7521
+ 2021-07-15 22:31:08,233 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7522
+ 2021-07-15 22:31:08,233 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7523
+ 2021-07-15 22:31:08,234 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7524
+ 2021-07-15 22:31:08,917 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7525
+ 2021-07-15 22:31:08,917 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7526
+ 2021-07-15 22:31:09,918 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7527
+ 2021-07-15 22:31:11,918 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7528
+ 2021-07-15 22:31:13,919 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7529
+ 2021-07-15 22:31:15,920 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7530
+ 2021-07-15 22:31:17,921 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7531
+ 2021-07-15 22:31:19,064 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7532
+ 2021-07-15 22:31:22,040 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7533
+ 2021-07-15 22:31:22,040 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7534
+ 2021-07-15 22:31:28,269 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7535
+ 2021-07-15 22:31:28,269 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7536
+ 2021-07-15 22:31:28,270 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7537
+ 2021-07-15 22:31:28,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7538
+ 2021-07-15 22:31:28,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7539
+ 2021-07-15 22:31:29,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7540
+ 2021-07-15 22:31:31,926 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7541
+ 2021-07-15 22:31:33,927 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7542
+ 2021-07-15 22:31:35,928 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7543
+ 2021-07-15 22:31:37,206 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7544
+ 2021-07-15 22:31:37,206 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7545
+ 2021-07-15 22:31:37,929 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7546
+ 2021-07-15 22:31:48,313 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7547
+ 2021-07-15 22:31:48,313 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7548
+ 2021-07-15 22:31:48,314 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7549
+ 2021-07-15 22:31:48,933 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7550
+ 2021-07-15 22:31:48,933 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7551
+ 2021-07-15 22:31:49,142 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7552
+ 2021-07-15 22:31:49,933 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7553
+ 2021-07-15 22:31:51,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7554
+ 2021-07-15 22:31:52,339 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7555
+ 2021-07-15 22:31:52,339 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7556
+ 2021-07-15 22:31:53,935 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7557
+ 2021-07-15 22:31:55,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7558
+ 2021-07-15 22:31:57,937 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7559
+ 2021-07-15 22:32:07,475 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7560
+ 2021-07-15 22:32:07,475 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7561
+ 2021-07-15 22:32:08,395 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7562
+ 2021-07-15 22:32:08,396 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7563
+ 2021-07-15 22:32:08,396 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7564
+ 2021-07-15 22:32:08,941 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7565
+ 2021-07-15 22:32:09,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7566
+ 2021-07-15 22:32:10,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7567
+ 2021-07-15 22:32:11,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7568
+ 2021-07-15 22:32:13,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7569
+ 2021-07-15 22:32:15,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7570
+ 2021-07-15 22:32:19,220 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7571
+ 2021-07-15 22:32:22,607 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7572
+ 2021-07-15 22:32:22,608 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7573
+ 2021-07-15 22:32:28,432 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7574
+ 2021-07-15 22:32:28,433 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7575
+ 2021-07-15 22:32:28,433 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7576
+ 2021-07-15 22:32:28,949 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7577
+ 2021-07-15 22:32:29,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7578
+ 2021-07-15 22:32:30,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7579
+ 2021-07-15 22:32:31,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7580
+ 2021-07-15 22:32:33,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7581
+ 2021-07-15 22:32:35,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7582
+ 2021-07-15 22:32:37,739 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7583
+ 2021-07-15 22:32:37,739 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7584
+ 2021-07-15 22:32:37,953 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7585
+ 2021-07-15 22:32:48,483 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7586
+ 2021-07-15 22:32:48,483 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7587
+ 2021-07-15 22:32:48,483 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7588
+ 2021-07-15 22:32:48,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7589
+ 2021-07-15 22:32:48,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7590
+ 2021-07-15 22:32:49,298 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7591
+ 2021-07-15 22:32:49,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7592
+ 2021-07-15 22:32:51,958 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7593
+ 2021-07-15 22:32:52,870 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7594
+ 2021-07-15 22:32:52,870 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7595
+ 2021-07-15 22:32:53,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7596
+ 2021-07-15 22:32:55,960 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7597
+ 2021-07-15 22:32:57,961 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7598
+ 2021-07-15 22:33:08,001 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7599
+ 2021-07-15 22:33:08,002 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7600
+ 2021-07-15 22:33:08,521 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7601
+ 2021-07-15 22:33:08,521 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7602
+ 2021-07-15 22:33:08,521 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7603
+ 2021-07-15 22:33:08,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7604
+ 2021-07-15 22:33:08,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7605
+ 2021-07-15 22:33:09,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7606
+ 2021-07-15 22:33:11,967 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7607
+ 2021-07-15 22:33:13,968 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7608
+ 2021-07-15 22:33:15,969 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7609
+ 2021-07-15 22:33:17,970 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7610
+ 2021-07-15 22:33:19,374 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7611
+ 2021-07-15 22:33:23,135 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7612
+ 2021-07-15 22:33:23,136 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7613
+ 2021-07-15 22:33:28,559 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7614
+ 2021-07-15 22:33:28,559 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7615
+ 2021-07-15 22:33:28,560 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7616
+ 2021-07-15 22:33:28,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7617
+ 2021-07-15 22:33:28,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7618
+ 2021-07-15 22:33:29,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7619
+ 2021-07-15 22:33:31,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7620
+ 2021-07-15 22:33:33,976 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7621
+ 2021-07-15 22:33:35,977 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7622
+ 2021-07-15 22:33:37,978 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7623
+ 2021-07-15 22:33:38,266 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7624
+ 2021-07-15 22:33:38,267 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7625
+ 2021-07-15 22:33:48,613 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7626
+ 2021-07-15 22:33:48,614 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7627
+ 2021-07-15 22:33:48,614 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7628
+ 2021-07-15 22:33:48,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7629
+ 2021-07-15 22:33:48,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7630
+ 2021-07-15 22:33:49,452 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7631
+ 2021-07-15 22:33:49,983 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7632
+ 2021-07-15 22:33:51,984 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7633
+ 2021-07-15 22:33:53,401 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7634
+ 2021-07-15 22:33:53,401 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7635
+ 2021-07-15 22:33:53,985 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7636
+ 2021-07-15 22:33:55,985 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7637
+ 2021-07-15 22:33:57,986 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7638
+ 2021-07-15 22:34:08,531 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7639
+ 2021-07-15 22:34:08,532 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7640
+ 2021-07-15 22:34:08,663 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7641
+ 2021-07-15 22:34:08,663 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7642
+ 2021-07-15 22:34:08,664 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7643
+ 2021-07-15 22:34:08,990 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7644
+ 2021-07-15 22:34:08,991 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7645
+ 2021-07-15 22:34:09,991 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7646
+ 2021-07-15 22:34:11,992 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7647
+ 2021-07-15 22:34:13,992 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7648
+ 2021-07-15 22:34:15,993 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7649
+ 2021-07-15 22:34:19,530 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7650
+ 2021-07-15 22:34:23,703 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7651
+ 2021-07-15 22:34:23,703 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7652
+ 2021-07-15 22:34:28,699 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7653
+ 2021-07-15 22:34:28,699 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7654
+ 2021-07-15 22:34:28,700 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7655
+ 2021-07-15 22:34:28,998 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7656
+ 2021-07-15 22:34:29,999 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7657
+ 2021-07-15 22:34:30,999 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7658
+ 2021-07-15 22:34:32,000 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7659
+ 2021-07-15 22:34:34,000 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7660
+ 2021-07-15 22:34:36,001 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7661
+ 2021-07-15 22:34:38,002 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7662
+ 2021-07-15 22:34:38,833 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7663
+ 2021-07-15 22:34:38,833 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7664
+ 2021-07-15 22:34:48,735 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7665
+ 2021-07-15 22:34:48,736 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7666
+ 2021-07-15 22:34:48,736 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7667
+ 2021-07-15 22:34:49,006 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7668
+ 2021-07-15 22:34:49,607 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7669
+ 2021-07-15 22:34:50,007 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7670
+ 2021-07-15 22:34:52,008 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7671
+ 2021-07-15 22:34:53,962 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7672
+ 2021-07-15 22:34:53,963 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7673
+ 2021-07-15 22:34:54,009 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7674
+ 2021-07-15 22:34:56,009 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7675
+ 2021-07-15 22:34:58,010 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7676
+ 2021-07-15 22:35:08,783 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7677
+ 2021-07-15 22:35:08,783 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7678
+ 2021-07-15 22:35:08,783 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7679
+ 2021-07-15 22:35:09,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7680
+ 2021-07-15 22:35:09,095 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7681
+ 2021-07-15 22:35:09,095 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7682
+ 2021-07-15 22:35:10,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7683
+ 2021-07-15 22:35:12,016 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7684
+ 2021-07-15 22:35:14,017 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7685
+ 2021-07-15 22:35:16,018 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7686
+ 2021-07-15 22:35:18,018 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7687
+ 2021-07-15 22:35:19,686 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7688
+ 2021-07-15 22:35:24,233 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7689
+ 2021-07-15 22:35:24,234 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7690
+ 2021-07-15 22:35:28,818 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7691
+ 2021-07-15 22:35:28,819 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7692
+ 2021-07-15 22:35:28,819 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7693
+ 2021-07-15 22:35:29,023 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7694
+ 2021-07-15 22:35:30,023 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7695
+ 2021-07-15 22:35:32,024 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7696
+ 2021-07-15 22:35:34,025 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7697
+ 2021-07-15 22:35:36,026 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7698
+ 2021-07-15 22:35:38,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7699
+ 2021-07-15 22:35:39,364 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7700
+ 2021-07-15 22:35:39,364 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7701
+ 2021-07-15 22:35:48,891 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7702
+ 2021-07-15 22:35:48,891 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7703
+ 2021-07-15 22:35:48,891 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7704
+ 2021-07-15 22:35:49,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7705
+ 2021-07-15 22:35:49,763 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7706
+ 2021-07-15 22:35:50,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7707
+ 2021-07-15 22:35:52,032 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7708
+ 2021-07-15 22:35:54,033 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7709
+ 2021-07-15 22:35:54,530 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7710
+ 2021-07-15 22:35:54,531 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7711
+ 2021-07-15 22:35:56,034 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7712
+ 2021-07-15 22:35:58,034 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7713
+ 2021-07-15 22:36:08,929 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7714
+ 2021-07-15 22:36:08,929 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7715
+ 2021-07-15 22:36:08,930 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7716
+ 2021-07-15 22:36:09,039 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7717
+ 2021-07-15 22:36:09,740 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7718
+ 2021-07-15 22:36:09,740 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7719
+ 2021-07-15 22:36:10,040 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7720
+ 2021-07-15 22:36:12,040 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7721
+ 2021-07-15 22:36:14,041 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7722
+ 2021-07-15 22:36:16,042 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7723
+ 2021-07-15 22:36:19,841 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7724
+ 2021-07-15 22:36:24,904 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7725
+ 2021-07-15 22:36:24,904 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7726
+ 2021-07-15 22:36:28,966 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7727
+ 2021-07-15 22:36:28,966 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7728
+ 2021-07-15 22:36:28,967 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7729
+ 2021-07-15 22:36:29,047 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7730
+ 2021-07-15 22:36:30,048 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7731
+ 2021-07-15 22:36:32,049 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7732
+ 2021-07-15 22:36:34,049 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7733
+ 2021-07-15 22:36:36,050 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7734
+ 2021-07-15 22:36:40,034 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7735
+ 2021-07-15 22:36:40,035 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7736
+ 2021-07-15 22:36:49,003 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7737
+ 2021-07-15 22:36:49,003 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7738
+ 2021-07-15 22:36:49,004 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7739
+ 2021-07-15 22:36:49,056 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7740
+ 2021-07-15 22:36:49,912 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7741
+ 2021-07-15 22:36:50,056 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7742
+ 2021-07-15 22:36:52,057 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7743
+ 2021-07-15 22:36:54,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7744
+ 2021-07-15 22:36:55,165 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7745
+ 2021-07-15 22:36:55,165 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7746
+ 2021-07-15 22:36:56,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7747
+ 2021-07-15 22:37:09,039 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7748
+ 2021-07-15 22:37:09,040 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7749
+ 2021-07-15 22:37:09,040 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7750
+ 2021-07-15 22:37:09,064 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7751
+ 2021-07-15 22:37:10,064 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7752
+ 2021-07-15 22:37:10,310 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7753
+ 2021-07-15 22:37:10,310 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7754
+ 2021-07-15 22:37:12,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7755
+ 2021-07-15 22:37:14,066 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7756
+ 2021-07-15 22:37:16,066 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7757
+ 2021-07-15 22:37:19,980 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7758
+ 2021-07-15 22:37:25,452 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7759
+ 2021-07-15 22:37:25,453 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7760
+ 2021-07-15 22:37:29,079 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7761
+ 2021-07-15 22:37:29,080 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7762
+ 2021-07-15 22:37:29,080 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7763
+ 2021-07-15 22:37:30,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7764
+ 2021-07-15 22:37:30,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7765
+ 2021-07-15 22:37:32,073 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7766
+ 2021-07-15 22:37:34,073 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7767
+ 2021-07-15 22:37:37,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7768
+ 2021-07-15 22:37:39,076 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7769
+ 2021-07-15 22:37:40,584 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7770
+ 2021-07-15 22:37:40,585 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7771
+ 2021-07-15 22:37:49,119 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7772
+ 2021-07-15 22:37:49,119 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7773
+ 2021-07-15 22:37:49,119 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7774
+ 2021-07-15 22:37:50,051 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7775
+ 2021-07-15 22:37:50,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7776
+ 2021-07-15 22:37:50,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7777
+ 2021-07-15 22:37:51,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7778
+ 2021-07-15 22:37:53,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7779
+ 2021-07-15 22:37:55,083 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7780
+ 2021-07-15 22:37:55,719 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7781
+ 2021-07-15 22:37:55,719 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7782
+ 2021-07-15 22:37:57,084 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7783
+ 2021-07-15 22:37:59,084 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7784
+ 2021-07-15 22:38:09,161 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7785
+ 2021-07-15 22:38:09,162 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7786
+ 2021-07-15 22:38:09,162 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7787
+ 2021-07-15 22:38:10,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7788
+ 2021-07-15 22:38:10,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7789
+ 2021-07-15 22:38:10,879 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7790
+ 2021-07-15 22:38:10,880 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7791
+ 2021-07-15 22:38:11,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7792
+ 2021-07-15 22:38:13,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7793
+ 2021-07-15 22:38:15,091 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7794
+ 2021-07-15 22:38:17,092 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7795
+ 2021-07-15 22:38:19,092 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7796
+ 2021-07-15 22:38:20,126 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7797
+ 2021-07-15 22:38:26,024 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7798
+ 2021-07-15 22:38:26,024 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7799
+ 2021-07-15 22:38:29,201 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7800
+ 2021-07-15 22:38:29,201 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7801
+ 2021-07-15 22:38:29,202 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7802
+ 2021-07-15 22:38:30,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7803
+ 2021-07-15 22:38:30,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7804
+ 2021-07-15 22:38:31,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7805
+ 2021-07-15 22:38:33,098 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7806
+ 2021-07-15 22:38:35,099 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7807
+ 2021-07-15 22:38:37,100 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7808
+ 2021-07-15 22:38:39,101 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7809
+ 2021-07-15 22:38:41,155 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7810
+ 2021-07-15 22:38:41,155 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7811
+ 2021-07-15 22:38:49,246 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7812
+ 2021-07-15 22:38:49,246 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7813
+ 2021-07-15 22:38:49,247 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7814
+ 2021-07-15 22:38:50,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7815
+ 2021-07-15 22:38:50,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7816
+ 2021-07-15 22:38:50,203 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7817
+ 2021-07-15 22:38:51,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7818
+ 2021-07-15 22:38:53,107 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7819
+ 2021-07-15 22:38:55,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7820
+ 2021-07-15 22:38:56,287 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7821
+ 2021-07-15 22:38:56,287 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7822
+ 2021-07-15 22:38:57,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7823
+ 2021-07-15 22:39:09,286 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7824
+ 2021-07-15 22:39:09,287 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7825
+ 2021-07-15 22:39:09,287 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7826
+ 2021-07-15 22:39:10,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7827
+ 2021-07-15 22:39:10,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7828
+ 2021-07-15 22:39:11,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7829
+ 2021-07-15 22:39:11,422 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7830
+ 2021-07-15 22:39:11,422 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7831
+ 2021-07-15 22:39:13,115 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7832
+ 2021-07-15 22:39:15,116 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7833
+ 2021-07-15 22:39:17,117 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7834
+ 2021-07-15 22:39:20,282 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7835
+ 2021-07-15 22:39:26,559 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7836
+ 2021-07-15 22:39:26,560 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7837
+ 2021-07-15 22:39:29,325 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7838
+ 2021-07-15 22:39:29,325 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7839
+ 2021-07-15 22:39:29,326 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7840
+ 2021-07-15 22:39:30,122 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7841
+ 2021-07-15 22:39:31,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7842
+ 2021-07-15 22:39:33,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7843
+ 2021-07-15 22:39:35,125 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7844
+ 2021-07-15 22:39:37,126 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7845
+ 2021-07-15 22:39:41,700 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7846
+ 2021-07-15 22:39:41,700 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7847
+ 2021-07-15 22:39:49,363 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7848
+ 2021-07-15 22:39:49,364 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7849
+ 2021-07-15 22:39:49,364 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7850
+ 2021-07-15 22:39:50,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7851
+ 2021-07-15 22:39:50,365 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7852
+ 2021-07-15 22:39:51,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7853
+ 2021-07-15 22:39:53,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7854
+ 2021-07-15 22:39:55,133 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7855
+ 2021-07-15 22:39:56,832 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7856
+ 2021-07-15 22:39:56,833 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7857
+ 2021-07-15 22:39:57,134 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7858
+ 2021-07-15 22:39:59,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7859
+ 2021-07-15 22:40:09,402 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7860
+ 2021-07-15 22:40:09,403 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7861
+ 2021-07-15 22:40:09,403 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7862
+ 2021-07-15 22:40:10,139 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7863
+ 2021-07-15 22:40:11,140 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7864
+ 2021-07-15 22:40:11,970 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7865
+ 2021-07-15 22:40:11,970 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7866
+ 2021-07-15 22:40:13,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7867
+ 2021-07-15 22:40:15,142 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7868
+ 2021-07-15 22:40:17,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7869
+ 2021-07-15 22:40:19,144 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7870
+ 2021-07-15 22:40:20,463 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7871
+ 2021-07-15 22:40:27,103 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7872
+ 2021-07-15 22:40:27,103 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7873
+ 2021-07-15 22:40:29,440 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7874
+ 2021-07-15 22:40:29,440 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7875
+ 2021-07-15 22:40:29,441 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7876
+ 2021-07-15 22:40:30,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7877
+ 2021-07-15 22:40:31,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7878
+ 2021-07-15 22:40:33,150 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7879
+ 2021-07-15 22:40:35,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7880
+ 2021-07-15 22:40:37,152 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7881
+ 2021-07-15 22:40:39,153 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7882
+ 2021-07-15 22:40:42,307 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7883
+ 2021-07-15 22:40:42,308 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7884
+ 2021-07-15 22:40:49,519 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7885
+ 2021-07-15 22:40:49,519 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7886
+ 2021-07-15 22:40:49,520 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7887
+ 2021-07-15 22:40:50,157 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7888
+ 2021-07-15 22:40:50,539 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7889
+ 2021-07-15 22:40:51,158 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7890
+ 2021-07-15 22:40:53,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7891
+ 2021-07-15 22:40:55,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7892
+ 2021-07-15 22:40:57,160 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7893
+ 2021-07-15 22:40:57,494 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7894
+ 2021-07-15 22:40:57,494 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7895
+ 2021-07-15 22:40:59,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7896
+ 2021-07-15 22:41:09,616 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7897
+ 2021-07-15 22:41:09,617 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7898
+ 2021-07-15 22:41:09,618 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7899
+ 2021-07-15 22:41:10,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7900
+ 2021-07-15 22:41:11,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7901
+ 2021-07-15 22:41:12,624 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7902
+ 2021-07-15 22:41:12,624 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7903
+ 2021-07-15 22:41:13,166 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7904
+ 2021-07-15 22:41:15,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7905
+ 2021-07-15 22:41:17,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7906
+ 2021-07-15 22:41:19,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7907
+ 2021-07-15 22:41:20,618 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7908
+ 2021-07-15 22:41:27,754 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7909
+ 2021-07-15 22:41:27,755 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7910
+ 2021-07-15 22:41:29,669 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7911
+ 2021-07-15 22:41:29,669 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7912
+ 2021-07-15 22:41:29,670 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7913
+ 2021-07-15 22:41:30,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7914
+ 2021-07-15 22:41:31,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7915
+ 2021-07-15 22:41:33,174 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7916
+ 2021-07-15 22:41:35,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7917
+ 2021-07-15 22:41:37,176 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7918
+ 2021-07-15 22:41:39,177 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7919
+ 2021-07-15 22:41:42,886 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7920
+ 2021-07-15 22:41:42,886 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7921
+ 2021-07-15 22:41:49,725 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7922
+ 2021-07-15 22:41:49,725 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7923
+ 2021-07-15 22:41:49,726 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7924
+ 2021-07-15 22:41:50,181 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7925
+ 2021-07-15 22:41:50,696 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7926
+ 2021-07-15 22:41:51,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7927
+ 2021-07-15 22:41:53,183 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7928
+ 2021-07-15 22:41:55,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7929
+ 2021-07-15 22:41:57,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7930
+ 2021-07-15 22:41:58,017 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7931
+ 2021-07-15 22:41:58,017 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7932
+ 2021-07-15 22:42:09,776 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7933
+ 2021-07-15 22:42:09,776 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7934
+ 2021-07-15 22:42:09,777 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7935
+ 2021-07-15 22:42:10,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7936
+ 2021-07-15 22:42:11,190 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7937
+ 2021-07-15 22:42:13,150 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7938
+ 2021-07-15 22:42:13,150 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7939
+ 2021-07-15 22:42:13,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7940
+ 2021-07-15 22:42:15,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7941
+ 2021-07-15 22:42:17,192 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7942
+ 2021-07-15 22:42:20,774 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7943
+ 2021-07-15 22:42:28,282 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7944
+ 2021-07-15 22:42:28,282 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7945
+ 2021-07-15 22:42:29,826 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7946
+ 2021-07-15 22:42:29,826 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7947
+ 2021-07-15 22:42:29,826 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7948
+ 2021-07-15 22:42:30,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7949
+ 2021-07-15 22:42:31,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7950
+ 2021-07-15 22:42:33,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7951
+ 2021-07-15 22:42:35,199 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7952
+ 2021-07-15 22:42:37,200 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7953
+ 2021-07-15 22:42:39,201 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7954
+ 2021-07-15 22:42:43,418 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7955
+ 2021-07-15 22:42:43,418 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7956
+ 2021-07-15 22:42:49,861 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7957
+ 2021-07-15 22:42:49,862 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7958
+ 2021-07-15 22:42:49,862 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7959
+ 2021-07-15 22:42:50,205 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7960
+ 2021-07-15 22:42:50,851 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7961
+ 2021-07-15 22:42:51,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7962
+ 2021-07-15 22:42:53,207 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7963
+ 2021-07-15 22:42:55,207 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7964
+ 2021-07-15 22:42:57,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7965
+ 2021-07-15 22:42:58,552 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7966
+ 2021-07-15 22:42:58,552 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7967
+ 2021-07-15 22:42:59,209 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7968
+ 2021-07-15 22:43:09,897 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7969
+ 2021-07-15 22:43:09,898 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7970
+ 2021-07-15 22:43:09,898 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7971
+ 2021-07-15 22:43:10,214 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7972
+ 2021-07-15 22:43:11,214 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7973
+ 2021-07-15 22:43:13,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7974
+ 2021-07-15 22:43:13,686 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7975
+ 2021-07-15 22:43:13,686 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7976
+ 2021-07-15 22:43:15,216 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7977
+ 2021-07-15 22:43:17,217 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7978
+ 2021-07-15 22:43:19,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7979
+ 2021-07-15 22:43:20,930 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7980
+ 2021-07-15 22:43:28,817 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7981
+ 2021-07-15 22:43:28,818 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7982
+ 2021-07-15 22:43:29,934 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7983
+ 2021-07-15 22:43:29,934 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7984
+ 2021-07-15 22:43:29,935 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7985
+ 2021-07-15 22:43:30,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7986
+ 2021-07-15 22:43:31,223 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7987
+ 2021-07-15 22:43:33,223 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7988
+ 2021-07-15 22:43:35,224 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7989
+ 2021-07-15 22:43:37,225 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7990
+ 2021-07-15 22:43:39,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7991
+ 2021-07-15 22:43:43,951 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
7992
+ 2021-07-15 22:43:43,952 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
7993
+ 2021-07-15 22:43:49,971 DEBUG SenderThread:735392 [sender.py:send():179] send: history
7994
+ 2021-07-15 22:43:49,971 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
7995
+ 2021-07-15 22:43:49,972 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
7996
+ 2021-07-15 22:43:50,230 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
7997
+ 2021-07-15 22:43:51,008 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
7998
+ 2021-07-15 22:43:51,231 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
7999
+ 2021-07-15 22:43:53,232 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8000
+ 2021-07-15 22:43:55,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8001
+ 2021-07-15 22:43:57,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8002
+ 2021-07-15 22:43:59,091 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8003
+ 2021-07-15 22:43:59,091 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8004
+ 2021-07-15 22:43:59,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8005
+ 2021-07-15 22:44:10,006 DEBUG SenderThread:735392 [sender.py:send():179] send: history
8006
+ 2021-07-15 22:44:10,007 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
8007
+ 2021-07-15 22:44:10,007 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
8008
+ 2021-07-15 22:44:10,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
8009
+ 2021-07-15 22:44:11,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8010
+ 2021-07-15 22:44:13,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8011
+ 2021-07-15 22:44:14,220 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8012
+ 2021-07-15 22:44:14,221 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8013
+ 2021-07-15 22:44:15,241 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8014
+ 2021-07-15 22:44:17,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8015
+ 2021-07-15 22:44:19,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8016
+ 2021-07-15 22:44:21,085 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
8017
+ 2021-07-15 22:44:29,354 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8018
+ 2021-07-15 22:44:29,354 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8019
+ 2021-07-15 22:44:30,044 DEBUG SenderThread:735392 [sender.py:send():179] send: history
8020
+ 2021-07-15 22:44:30,045 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
8021
+ 2021-07-15 22:44:30,046 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
8022
+ 2021-07-15 22:44:30,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
8023
+ 2021-07-15 22:44:31,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8024
+ 2021-07-15 22:44:33,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8025
+ 2021-07-15 22:44:35,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8026
+ 2021-07-15 22:44:37,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8027
+ 2021-07-15 22:44:44,487 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8028
+ 2021-07-15 22:44:44,487 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8029
+ 2021-07-15 22:44:50,085 DEBUG SenderThread:735392 [sender.py:send():179] send: history
8030
+ 2021-07-15 22:44:50,086 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
8031
+ 2021-07-15 22:44:50,086 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
8032
+ 2021-07-15 22:44:50,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
8033
+ 2021-07-15 22:44:51,163 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
8034
+ 2021-07-15 22:44:51,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8035
+ 2021-07-15 22:44:53,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8036
+ 2021-07-15 22:44:55,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8037
+ 2021-07-15 22:44:57,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8038
+ 2021-07-15 22:44:59,618 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8039
+ 2021-07-15 22:44:59,618 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8040
+ 2021-07-15 22:45:10,131 DEBUG SenderThread:735392 [sender.py:send():179] send: history
8041
+ 2021-07-15 22:45:10,132 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
8042
+ 2021-07-15 22:45:10,132 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
8043
+ 2021-07-15 22:45:10,263 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
8044
+ 2021-07-15 22:45:11,263 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8045
+ 2021-07-15 22:45:13,264 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8046
+ 2021-07-15 22:45:14,750 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8047
+ 2021-07-15 22:45:14,750 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8048
+ 2021-07-15 22:45:15,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8049
+ 2021-07-15 22:45:17,266 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8050
+ 2021-07-15 22:45:21,242 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
8051
+ 2021-07-15 22:45:29,881 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8052
+ 2021-07-15 22:45:29,881 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8053
+ 2021-07-15 22:45:30,179 DEBUG SenderThread:735392 [sender.py:send():179] send: history
8054
+ 2021-07-15 22:45:30,179 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
8055
+ 2021-07-15 22:45:30,180 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
8056
+ 2021-07-15 22:45:30,271 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
8057
+ 2021-07-15 22:45:31,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8058
+ 2021-07-15 22:45:32,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8059
+ 2021-07-15 22:45:34,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8060
+ 2021-07-15 22:45:36,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8061
+ 2021-07-15 22:45:38,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8062
+ 2021-07-15 22:45:45,014 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8063
+ 2021-07-15 22:45:45,015 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8064
+ 2021-07-15 22:45:50,227 DEBUG SenderThread:735392 [sender.py:send():179] send: history
8065
+ 2021-07-15 22:45:50,227 DEBUG SenderThread:735392 [sender.py:send():179] send: summary
8066
+ 2021-07-15 22:45:50,228 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
8067
+ 2021-07-15 22:45:50,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json
8068
+ 2021-07-15 22:45:51,281 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8069
+ 2021-07-15 22:45:51,319 DEBUG SenderThread:735392 [sender.py:send():179] send: stats
8070
+ 2021-07-15 22:45:52,281 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8071
+ 2021-07-15 22:45:54,282 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8072
+ 2021-07-15 22:45:56,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8073
+ 2021-07-15 22:45:58,284 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
8074
+ 2021-07-15 22:46:00,146 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status
8075
+ 2021-07-15 22:46:00,147 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status
8076
+ 2021-07-15 22:46:00,285 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log
wandb/run-20210715_185845-dq8uirtg/run-dq8uirtg.wandb CHANGED
Binary files a/wandb/run-20210715_185845-dq8uirtg/run-dq8uirtg.wandb and b/wandb/run-20210715_185845-dq8uirtg/run-dq8uirtg.wandb differ