Saving weights and logs of step 240000

checkpoint_240000

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829a6abc0c8b1c376c211be682dc46f6a7d7fbc1ddf4114262e4e1cf169cde42
+size 1530270447

create_pt_model.py

@@ -2,4 +2,4 @@ from transformers import BigBirdForMaskedLM
 import torch
 
 pt_model = BigBirdForMaskedLM.from_pretrained(".", from_flax=True)
-pt_model.save_pretrained(".")
+pt_model.save_pretrained(".",save_config=False)

events.out.tfevents.1626397185.t1v-n-f5c06ea1-w-0.760449.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55333c246f74b2c84620675b6048997ae79474af6bfdd07693a18bed98c164cc
-size 11281744
+oid sha256:c1c6fce6e15405613b7ad5d5c8a11bab2c7731406ba245e3113f9ef8e63e75f6
+size 11424694

events.out.tfevents.1626429561.t1v-n-f5c06ea1-w-0.782479.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45a436bec44dfea2809d645a06db2ef771283532bf125612badb26de71cec05c
+size 2256466

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8f5ca7b60c5e4b03edfdde3f172ec2572f02f5753f13fe7a658c57c2fc9bae5
+oid sha256:0ae1d45c74c0892af03af2f6156aa1d6c21cff644f407de47ad5b18a10b3c17c
 size 510090043

run.sh

@@ -16,7 +16,7 @@ python ./run_mlm_flax_no_accum.py \
     --adam_beta2="0.98" \
     --logging_steps="50" \
     --eval_steps="6000" \
-    --num_train_epochs="3"\
+    --num_train_epochs="5"\
     --preprocessing_num_workers="96" \
     --save_steps="15000" \
     --learning_rate="3e-5" \

wandb/debug-internal.log

@@ -1 +1 @@
-run-20210716_005946-3bte92ft/logs/debug-internal.log
+run-20210716_095921-13hxxunp/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20210716_005946-3bte92ft/logs/debug.log
+run-20210716_095921-13hxxunp/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20210716_005946-3bte92ft
+run-20210716_095921-13hxxunp

wandb/run-20210716_005946-3bte92ft/files/output.log

@@ -8608,3 +8608,123 @@ tcmalloc: large alloc 2715181056 bytes == 0x41e66a000 @  0x7f779c64f680 0x7f779c
 [09:50:33] - INFO - absl - Saved checkpoint at checkpoint_225000
 [09:50:34] - INFO - huggingface_hub.repository - git version 2.25.1
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
+[09:50:35] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[09:51:44] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 2.1 GB | 46 MB/s, done.
+
+
+
+
+Training...: 75049it [8:46:15,  2.69it/s]
+
+
+
+
+
+Training...: 75099it [8:46:35,  2.70it/s]
+
+
+
+
+
+Training...: 75149it [8:46:55,  2.72it/s]
+
+
+
+
+
+Training...: 75199it [8:47:15,  2.66it/s]
+
+
+
+
+
+Training...: 75249it [8:47:35,  2.68it/s]
+
+
+
+
+
+Training...: 75299it [8:47:55,  2.69it/s]
+
+
+
+
+
+Training...: 75349it [8:48:15,  2.68it/s]
+
+
+
+
+
+Training...: 75399it [8:48:35,  2.72it/s]
+
+
+
+
+
+Training...: 75449it [8:48:55,  2.69it/s]
+
+
+
+
+
+Training...: 75499it [8:49:15,  2.70it/s]
+
+
+
+
+
+Training...: 75549it [8:49:36,  2.68it/s]
+
+
+
+
+
+Training...: 75599it [8:49:56,  2.68it/s]
+
+
+
+
+
+Training...: 75649it [8:50:16,  2.72it/s]
+
+
+
+
+
+Training...: 75699it [8:50:36,  2.66it/s]
+
+
+
+
+
+Training...: 75749it [8:50:56,  2.72it/s]
+
+
+
+
+
+Training...: 75799it [8:51:16,  2.70it/s]
+
+
+
+
+
+Training...: 75849it [8:51:36,  2.66it/s]
+
+
+
+
+
+Training...: 75899it [8:51:56,  2.75it/s]
+
+
+
+
+
+Training...: 75949it [8:52:16,  2.67it/s]
+
+
+
+
+

wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 225000, "learning_rate": 2.3847169359214604e-05, "train_loss": 2.061807155609131, "_runtime": 31840, "_timestamp": 1626429026, "_step": 1511, "eval_step": 222000, "eval_accuracy": 0.6153644919395447, "eval_loss": 1.9865816831588745}
+{"training_step": 225950, "learning_rate": 2.3819982743589208e-05, "train_loss": 1.9876642227172852, "_runtime": 32300, "_timestamp": 1626429486, "_step": 1530, "eval_step": 222000, "eval_accuracy": 0.6153644919395447, "eval_loss": 1.9865816831588745}

wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log

@@ -19048,3 +19048,272 @@
 2021-07-16 09:50:37,587 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-16 09:50:37,588 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
 2021-07-16 09:50:38,066 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:50:52,724 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:50:52,724 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:51:04,008 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:51:07,872 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:51:07,872 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:51:23,004 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:51:23,004 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:51:34,084 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:51:38,251 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:51:38,252 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:51:46,091 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:51:48,092 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:51:50,093 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:51:52,094 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:51:53,384 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:51:53,384 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:51:54,094 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:04,156 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:52:05,407 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:52:05,407 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:52:05,408 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:52:06,099 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:06,099 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:52:08,099 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:08,515 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:52:08,515 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:52:10,100 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:13,101 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:15,102 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:23,654 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:52:23,655 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:52:25,445 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:52:25,445 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:52:25,446 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:52:26,106 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:26,106 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:52:27,107 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:29,107 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:31,108 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:33,109 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:34,233 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:52:35,110 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:38,789 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:52:38,789 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:52:45,483 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:52:45,484 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:52:45,484 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:52:46,114 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:46,114 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:52:47,114 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:49,115 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:51,116 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:53,117 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:52:53,952 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:52:53,952 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:52:55,117 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:04,307 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:53:05,595 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:53:05,596 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:53:05,596 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:53:06,121 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:53:07,122 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:09,081 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:53:09,082 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:53:09,123 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:11,124 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:13,124 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:15,125 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:24,213 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:53:24,214 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:53:25,635 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:53:25,635 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:53:25,636 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:53:26,130 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:53:27,130 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:29,131 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:31,132 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:33,132 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:34,383 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:53:35,133 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:39,343 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:53:39,343 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:53:45,684 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:53:45,685 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:53:45,685 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:53:46,138 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:53:47,138 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:49,139 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:51,140 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:53,141 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:53:54,471 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:53:54,471 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:53:55,142 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:04,461 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:54:05,722 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:54:05,722 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:54:05,723 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:54:06,146 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:54:07,146 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:09,147 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:09,602 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:54:09,603 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:54:11,148 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:13,149 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:15,150 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:24,736 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:54:24,736 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:54:25,763 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:54:25,763 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:54:25,764 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:54:26,154 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:54:27,154 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:29,155 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:31,156 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:33,157 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:34,537 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:54:35,158 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:39,870 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:54:39,870 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:54:45,806 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:54:45,806 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:54:45,807 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:54:46,162 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:54:47,163 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:49,164 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:51,164 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:53,165 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:54:55,001 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:54:55,001 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:54:55,166 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:04,608 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:55:05,846 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:55:05,846 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:55:05,847 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:55:06,171 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:55:07,171 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:09,172 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:10,134 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:55:10,134 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:55:11,173 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:13,174 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:15,175 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:25,266 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:55:25,267 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:55:25,884 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:55:25,884 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:55:25,885 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:55:26,180 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:55:27,180 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:29,181 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:31,182 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:33,183 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:34,684 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:55:35,184 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:40,396 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:55:40,396 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:55:45,925 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:55:45,925 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:55:45,926 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:55:46,189 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:55:47,189 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:49,190 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:51,191 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:53,192 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:55,193 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:55:55,529 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:55:55,530 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:56:04,762 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:56:05,966 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:56:05,966 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:56:05,967 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:56:06,197 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:56:07,197 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:09,198 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:10,664 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:56:10,664 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:56:11,199 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:13,200 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:15,201 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:25,796 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:56:25,796 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:56:26,066 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:56:26,066 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:56:26,067 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:56:26,205 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:56:27,205 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:29,206 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:31,207 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:33,208 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:34,837 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:56:35,209 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:40,928 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:56:40,928 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:56:46,118 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:56:46,118 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:56:46,119 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:56:46,213 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:56:47,214 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:49,215 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:51,215 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:53,216 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:55,217 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:56:56,061 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:56:56,061 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:57:04,910 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:57:06,160 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:57:06,160 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:57:06,160 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:57:06,221 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:57:07,222 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:09,223 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:11,191 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:57:11,191 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:57:11,224 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:13,224 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:15,225 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:26,198 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:57:26,198 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:57:26,199 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:57:26,230 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:57:26,613 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:57:26,613 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:57:27,230 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:29,231 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:31,232 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:33,233 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:34,985 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:57:35,234 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:41,780 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:57:41,780 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:57:46,293 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:57:46,293 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:57:46,294 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:57:47,239 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:47,239 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:57:49,240 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:51,240 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:53,241 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:55,242 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:57:56,915 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:57:56,915 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:58:05,063 DEBUG   SenderThread:761704 [sender.py:send():179] send: stats
+2021-07-16 09:58:06,338 DEBUG   SenderThread:761704 [sender.py:send():179] send: history
+2021-07-16 09:58:06,339 DEBUG   SenderThread:761704 [sender.py:send():179] send: summary
+2021-07-16 09:58:06,339 INFO    SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 09:58:07,247 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:58:07,247 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
+2021-07-16 09:58:09,247 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:58:11,248 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:58:12,051 DEBUG   HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 09:58:12,051 DEBUG   SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 09:58:13,249 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:58:15,250 INFO    Thread-8  :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
+2021-07-16 09:58:23,978 WARNING MainThread:761704 [internal.py:wandb_internal():147] Internal process interrupt: 1
+2021-07-16 09:58:24,417 WARNING MainThread:761704 [internal.py:wandb_internal():147] Internal process interrupt: 2
+2021-07-16 09:58:24,417 ERROR   MainThread:761704 [internal.py:wandb_internal():150] Internal process interrupted.
+2021-07-16 09:58:25,070 INFO    WriterThread:761704 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb
+2021-07-16 09:58:25,188 INFO    SenderThread:761704 [sender.py:finish():945] shutting down sender
+2021-07-16 09:58:25,188 INFO    SenderThread:761704 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-16 09:58:25,191 INFO    HandlerThread:761704 [handler.py:finish():638] shutting down handler
+2021-07-16 09:58:25,253 INFO    SenderThread:761704 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files
+2021-07-16 09:58:25,253 INFO    SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/requirements.txt requirements.txt
+2021-07-16 09:58:25,253 INFO    SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log output.log
+2021-07-16 09:58:25,254 INFO    SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-metadata.json wandb-metadata.json
+2021-07-16 09:58:25,254 INFO    SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/config.yaml config.yaml
+2021-07-16 09:58:25,254 INFO    SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json wandb-summary.json
+2021-07-16 09:58:25,254 INFO    SenderThread:761704 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-16 09:58:25,254 INFO    SenderThread:761704 [file_pusher.py:join():182] waiting for file pusher
+2021-07-16 09:58:25,419 INFO    MainThread:761704 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210716_005946-3bte92ft/logs/debug.log

@@ -24,3 +24,5 @@ config: {}
 2021-07-16 00:59:48,518 INFO    MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul16_00-59-37_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 15000, 'save_total_limit': 20, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 6000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
 2021-07-16 00:59:48,520 INFO    MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
 2021-07-16 00:59:48,521 INFO    MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 4000}
+2021-07-16 09:58:24,335 INFO    MainThread:760449 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
+2021-07-16 09:58:24,336 INFO    MainThread:760449 [wandb_run.py:_restore():1565] restore

wandb/run-20210716_095921-13hxxunp/files/config.yaml

@@ -0,0 +1,308 @@
+wandb_version: 1
+
+__cached__setup_devices:
+  desc: null
+  value: cpu
+_n_gpu:
+  desc: null
+  value: 0
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 1
+      - 3
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.98
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: null
+dataset_name:
+  desc: null
+  value: null
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 6000
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 3.0e-05
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul16_09-59-13_t1v-n-f5c06ea1-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 50
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_eval_samples:
+  desc: null
+  value: 4000
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 4096
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: big_bird
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 5.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 1
+per_device_train_batch_size:
+  desc: null
+  value: 1
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: 96
+push_to_hub:
+  desc: null
+  value: true
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: ./
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 15000
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: 20
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 10000
+weight_decay:
+  desc: null
+  value: 0.0095

wandb/run-20210716_095921-13hxxunp/files/output.log

@@ -0,0 +1,1779 @@
+
+[09:59:36] - INFO - absl - Restoring checkpoint from ./checkpoint_225000
+tcmalloc: large alloc 1530273792 bytes == 0x9d58c000 @  0x7fe91a70f680 0x7fe91a730824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fe91a5240b3 0x5f96de
+/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/5):   0%|                                                                      | 0/5 [00:00<?, ?it/s]
+Training...: 0it [00:00, ?it/s]
+
+
+
+
+Training...: 49it [04:22,  2.65it/s]
+
+
+
+
+
+Training...: 100it [04:55,  4.01s/it]
+
+
+
+
+
+Training...: 152it [05:15,  2.24s/it]
+
+
+
+
+
+Training...: 202it [05:36,  2.25s/it]
+
+
+
+
+Training...: 249it [05:42,  2.68it/s]
+
+
+
+
+Training...: 299it [06:02,  2.67it/s]
+
+
+
+
+
+Training...: 350it [06:35,  4.01s/it]
+
+
+
+
+
+Training...: 400it [06:56,  4.01s/it]
+
+
+
+
+
+Training...: 450it [07:16,  4.01s/it]
+
+
+
+
+
+Training...: 502it [07:36,  2.24s/it]
+
+
+
+
+
+Training...: 552it [07:56,  2.25s/it]
+
+
+
+
+
+Training...: 604it [08:16,  1.39s/it]
+
+
+
+
+Training...: 649it [08:23,  2.67it/s]
+
+
+
+
+Training...: 699it [08:43,  2.69it/s]
+
+
+
+
+Training...: 749it [09:03,  2.67it/s]
+
+
+
+
+Training...: 799it [09:23,  2.68it/s]
+
+
+
+
+
+Training...: 850it [09:56,  4.01s/it]
+
+
+
+
+
+Training...: 900it [10:16,  4.01s/it]
+
+
+
+
+
+Training...: 950it [10:36,  4.01s/it]
+
+
+
+
+
+Training...: 1002it [10:56,  2.25s/it]
+
+
+
+
+
+Training...: 1054it [11:16,  1.39s/it]
+
+
+
+
+
+Training...: 1104it [11:36,  1.39s/it]
+
+
+
+
+Training...: 1149it [11:43,  2.69it/s]
+
+
+
+
+Training...: 1199it [12:03,  2.67it/s]
+
+
+
+
+Training...: 1249it [12:23,  2.67it/s]
+
+
+
+
+Training...: 1299it [12:43,  2.69it/s]
+
+
+
+
+Training...: 1349it [13:03,  2.67it/s]
+
+
+
+
+
+Training...: 1400it [13:36,  4.01s/it]
+
+
+
+
+
+Training...: 1450it [13:56,  4.01s/it]
+
+
+
+
+
+Training...: 1500it [14:16,  4.01s/it]
+
+
+
+
+
+Training...: 1552it [14:37,  2.24s/it]
+
+
+
+
+
+Training...: 1602it [14:57,  2.24s/it]
+
+
+
+
+
+Training...: 1654it [15:17,  1.39s/it]
+
+
+
+
+Training...: 1699it [15:23,  2.67it/s]
+
+
+
+
+Training...: 1749it [15:43,  2.70it/s]
+
+
+
+
+Training...: 1799it [16:03,  2.67it/s]
+
+
+
+
+
+Training...: 1850it [16:37,  4.01s/it]
+
+
+
+
+
+Training...: 1900it [16:57,  4.01s/it]
+
+
+
+
+
+Training...: 1952it [17:17,  2.24s/it]
+
+
+
+
+
+Training...: 2002it [17:37,  2.24s/it]
+
+
+
+
+
+Training...: 2054it [17:57,  1.39s/it]
+
+
+
+
+
+Training...: 2104it [18:17,  1.39s/it]
+
+
+
+
+Training...: 2149it [18:24,  2.66it/s]
+
+
+
+
+Training...: 2199it [18:44,  2.68it/s]
+
+
+
+
+
+Training...: 2250it [19:17,  4.01s/it]
+
+
+
+
+
+Training...: 2300it [19:37,  4.01s/it]
+
+
+
+
+
+Training...: 2352it [19:57,  2.24s/it]
+
+
+
+
+
+Training...: 2402it [20:17,  2.24s/it]
+
+
+
+
+
+Training...: 2454it [20:37,  1.39s/it]
+
+
+
+
+
+Training...: 2504it [20:57,  1.39s/it]
+
+
+
+
+Training...: 2549it [21:04,  2.67it/s]
+
+
+
+
+Training...: 2599it [21:24,  2.67it/s]
+
+
+
+
+Training...: 2649it [21:44,  2.67it/s]
+
+
+
+
+
+Training...: 2702it [22:17,  2.24s/it]
+
+
+
+
+
+Training...: 2754it [22:38,  1.39s/it]
+
+
+
+
+
+Training...: 2804it [22:58,  1.39s/it]
+
+
+
+
+
+Training...: 2854it [23:18,  1.39s/it]
+
+
+
+
+
+Training...: 2900it [23:37,  4.01s/it]
+
+
+
+
+
+Training...: 2950it [23:57,  4.01s/it]
+
+
+
+
+
+Training...: 2999it [24:17,  2.67it/s]
+Step... (228000 | Loss: 2.156348943710327, Learning Rate: 2.6271016395185143e-05)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...: 3049it [25:46,  2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3099it [26:06,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3166it [26:41,  5.39it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3208it [27:00,  1.59it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3258it [27:20,  1.59it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3299it [27:26,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3349it [27:47,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3399it [28:07,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3449it [28:27,  2.66it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3499it [28:47,  2.63it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3549it [29:07,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3599it [29:27,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3649it [29:47,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3699it [30:07,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3749it [30:27,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3799it [30:47,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3849it [31:07,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3899it [31:27,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 3949it [31:47,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 3999it [32:07,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4049it [32:27,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4099it [32:47,  2.63it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4149it [33:07,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4199it [33:27,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4249it [33:47,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4299it [34:07,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 4349it [34:27,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4399it [34:47,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4449it [35:07,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4499it [35:27,  2.69it/s]███████████████████████████████��██████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4549it [35:47,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4599it [36:07,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4649it [36:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 4699it [36:48,  2.74it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 4749it [37:08,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 4799it [37:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 4849it [37:48,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4899it [38:08,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4949it [38:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 4999it [38:48,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5049it [39:08,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5099it [39:28,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5149it [39:48,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5199it [40:08,  2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5249it [40:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5299it [40:48,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5349it [41:08,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5399it [41:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5449it [41:48,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5499it [42:08,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5549it [42:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5599it [42:48,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5649it [43:08,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 5699it [43:28,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5749it [43:48,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5799it [44:08,  2.66it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5849it [44:28,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5899it [44:48,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5949it [45:09,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 5999it [45:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6049it [45:49,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6099it [46:09,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6149it [46:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6199it [46:49,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6249it [47:09,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6299it [47:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6349it [47:49,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6399it [48:09,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6449it [48:29,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6499it [48:49,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6549it [49:09,  2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6599it [49:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6649it [49:49,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6699it [50:09,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6749it [50:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6799it [50:49,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6849it [51:09,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 6899it [51:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6949it [51:49,  2.73it/s]████████████████████████████████���█████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 6999it [52:09,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7049it [52:29,  2.71it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7099it [52:49,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7149it [53:09,  2.74it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7199it [53:29,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7249it [53:50,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7299it [54:10,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7349it [54:30,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7399it [54:50,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7449it [55:10,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7499it [55:30,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7549it [55:50,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7599it [56:10,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7649it [56:30,  2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7699it [56:50,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7749it [57:10,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7799it [57:30,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7849it [57:50,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7899it [58:10,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7949it [58:30,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 7999it [58:50,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8049it [59:10,  2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8099it [59:30,  2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8149it [59:50,  2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8199it [1:00:10,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8249it [1:00:30,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8299it [1:00:50,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8349it [1:01:10,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8399it [1:01:30,  2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8449it [1:01:50,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8499it [1:02:11,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8549it [1:02:31,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8599it [1:02:51,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8649it [1:03:11,  2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8699it [1:03:31,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8749it [1:03:51,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8799it [1:04:11,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8849it [1:04:31,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8899it [1:04:51,  2.70it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8949it [1:05:11,  2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+Training...: 8999it [1:05:31,  2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+Training...: 8999it [1:05:44,  2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...: 9050it [1:07:08,  4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9102it [1:07:28,  2.25s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9152it [1:07:48,  2.25s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9204it [1:08:08,  1.39s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9249it [1:08:15,  2.69it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9299it [1:08:35,  2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9349it [1:08:55,  2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9399it [1:09:15,  2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9449it [1:09:35,  2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9499it [1:09:55,  2.69it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9549it [1:10:15,  2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9602it [1:10:49,  2.24s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9654it [1:11:09,  1.39s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9704it [1:11:29,  1.39s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9749it [1:11:35,  2.69it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 9799it [1:11:55,  2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9850it [1:12:29,  4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9900it [1:12:49,  4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 9950it [1:13:09,  4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10002it [1:13:29,  2.24s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10052it [1:13:49,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10104it [1:14:09,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10154it [1:14:29,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10200it [1:14:49,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10254it [1:15:09,  1.38s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10304it [1:15:29,  1.40s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10356it [1:15:49,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10406it [1:16:09,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10456it [1:16:29,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10500it [1:16:49,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10550it [1:17:09,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10600it [1:17:29,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10652it [1:17:49,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10704it [1:18:10,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10756it [1:18:30,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10806it [1:18:50,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10850it [1:19:09,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10900it [1:19:29,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 10950it [1:19:49,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11002it [1:20:10,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11052it [1:20:30,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11104it [1:20:50,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11154it [1:21:10,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11204it [1:21:30,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11256it [1:21:50,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11302it [1:22:10,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11356it [1:22:30,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11406it [1:22:50,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11458it [1:23:10,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11500it [1:23:30,  4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11552it [1:23:50,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11602it [1:24:10,  2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11654it [1:24:30,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11704it [1:24:50,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11754it [1:25:10,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11804it [1:25:30,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11856it [1:25:51,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11906it [1:26:11,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11956it [1:26:31,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12008it [1:26:51,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12060it [1:27:11,  2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12102it [1:27:30,  2.24s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12154it [1:27:51,  1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12206it [1:28:11,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12256it [1:28:31,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12308it [1:28:51,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12358it [1:29:11,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12406it [1:29:31,  1.05it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12458it [1:29:51,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12508it [1:30:11,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12560it [1:30:31,  2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12610it [1:30:51,  2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12659it [1:31:12,  2.14it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12712it [1:31:32,  3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 12764it [1:31:52,  4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12806it [1:32:11,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12856it [1:32:31,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12908it [1:32:51,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 12958it [1:33:12,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13008it [1:33:32,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13058it [1:33:52,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13110it [1:34:12,  2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13156it [1:34:32,  1.09it/s]███████���███████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13206it [1:34:52,  1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 13249it [1:34:58,  2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13299it [1:35:18,  2.70it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13349it [1:35:38,  2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 13408it [1:36:12,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13460it [1:36:32,  2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13508it [1:36:52,  1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13562it [1:37:12,  3.10it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13612it [1:37:32,  3.05it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 13662it [1:37:52,  3.10it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 13714it [1:38:12,  4.15it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 13760it [1:38:32,  2.19it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 13814it [1:38:53,  4.15it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 13849it [1:38:59,  2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 13899it [1:39:19,  2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 13949it [1:39:39,  2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 14010it [1:40:12,  2.25it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 14060it [1:40:32,  2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 14112it [1:40:53,  3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 14162it [1:41:13,  3.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 14212it [1:41:33,  3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14262it [1:41:53,  3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14314it [1:42:13,  4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14364it [1:42:33,  4.17it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14414it [1:42:53,  4.15it/s]████████████████████��██████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14464it [1:43:13,  4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14512it [1:43:33,  3.07it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14564it [1:43:53,  4.10it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 14599it [1:43:59,  2.68it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14649it [1:44:19,  2.71it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14699it [1:44:39,  2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 14762it [1:45:13,  3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14814it [1:45:33,  4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14862it [1:45:53,  2.98it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 14899it [1:45:59,  2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 14949it [1:46:19,  2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 14999it [1:46:39,  2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+Training...: 14999it [1:46:53,  2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+tcmalloc: large alloc 1354776576 bytes == 0x37b2fe000 @  0x7fe91a70f680 0x7fe91a72fbdd 0x7fe900cc920d 0x7fe900cd7340 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd2bd3 0x7fe900cd31fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161 0x67e1df
+tcmalloc: large alloc 2715181056 bytes == 0x3cc702000 @  0x7fe91a70f680 0x7fe91a72fbdd 0x7fe900cc920d 0x7fe900cd7340 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd2bd3 0x7fe900cd31fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a
+tcmalloc: large alloc 1530273792 bytes == 0x321dba000 @  0x7fe91a70f680 0x7fe91a730824 0x5f7b11 0x7fe900cd2c6f 0x7fe900cd31fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fe91a5240b3 0x5f96de
+[11:53:20] - INFO - absl - Saved checkpoint at checkpoint_240000
+[11:53:21] - INFO - huggingface_hub.repository - git version 2.25.1
+git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)

wandb/run-20210716_095921-13hxxunp/files/requirements.txt

@@ -0,0 +1,95 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==21.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+charset-normalizer==2.0.1
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.7.0
+gast==0.4.0
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+google-pasta==0.2.0
+grpcio==1.34.1
+h5py==3.1.0
+huggingface-hub==0.0.12
+idna==3.2
+install==1.3.4
+jax==0.2.17
+jaxlib==0.1.68
+joblib==1.0.1
+keras-nightly==2.5.0.dev2021032900
+keras-preprocessing==1.1.2
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.19.5
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+pathtools==0.1.2
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+protobuf==3.17.3
+psutil==5.8.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.15.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.5.0
+tensorflow==2.5.0
+termcolor==1.1.0
+tokenizers==0.10.3
+toolz==0.11.1
+torch==1.9.0
+tqdm==4.61.2
+transformers==4.9.0.dev0
+typing-extensions==3.7.4.3
+urllib3==1.26.6
+wandb==0.10.33
+werkzeug==2.0.1
+wheel==0.36.2
+wrapt==1.12.1
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210716_095921-13hxxunp/files/wandb-metadata.json

@@ -0,0 +1,45 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-16T09:59:24.007461",
+    "startedAt": "2021-07-16T09:59:21.964690",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--push_to_hub",
+        "--output_dir=./",
+        "--model_type=big_bird",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--max_seq_length=4096",
+        "--weight_decay=0.0095",
+        "--warmup_steps=10000",
+        "--overwrite_output_dir",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--logging_steps=50",
+        "--eval_steps=6000",
+        "--num_train_epochs=5",
+        "--preprocessing_num_workers=96",
+        "--save_steps=15000",
+        "--learning_rate=3e-5",
+        "--per_device_train_batch_size=1",
+        "--per_device_eval_batch_size=1",
+        "--save_total_limit=20",
+        "--max_eval_samples=4000",
+        "--resume_from_checkpoint=./"
+    ],
+    "state": "running",
+    "program": "./run_mlm_flax_no_accum.py",
+    "codePath": "run_mlm_flax_no_accum.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/pino-roberta-base",
+        "commit": "0e50dbbb7c47cf4a8062023d210e1d841e9cebea"
+    },
+    "email": null,
+    "root": "/home/dat/pino-roberta-base",
+    "host": "t1v-n-f5c06ea1-w-0",
+    "username": "dat",
+    "executable": "/home/dat/pino/bin/python"
+}

wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training_step": 240000, "learning_rate": 2.606575253594201e-05, "train_loss": 1.9378972053527832, "_runtime": 6829, "_timestamp": 1626436391, "_step": 302, "eval_step": 240000, "eval_accuracy": 0.6196709871292114, "eval_loss": 1.9561585187911987}

wandb/run-20210716_095921-13hxxunp/logs/debug.log

@@ -0,0 +1,26 @@
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_setup.py:_flush():69] setting env: {}
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/logs/debug.log
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_init.py:init():370] calling init triggers
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [wandb_init.py:init():419] starting backend
+2021-07-16 09:59:21,966 INFO    MainThread:782479 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-16 09:59:22,014 INFO    MainThread:782479 [backend.py:ensure_launched():135] starting backend process...
+2021-07-16 09:59:22,062 INFO    MainThread:782479 [backend.py:ensure_launched():139] started backend process with pid: 783720
+2021-07-16 09:59:22,065 INFO    MainThread:782479 [wandb_init.py:init():424] backend started and connected
+2021-07-16 09:59:22,068 INFO    MainThread:782479 [wandb_init.py:init():472] updated telemetry
+2021-07-16 09:59:22,069 INFO    MainThread:782479 [wandb_init.py:init():491] communicating current version
+2021-07-16 09:59:22,707 INFO    MainThread:782479 [wandb_init.py:init():496] got version response upgrade_message: "wandb version 0.11.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2021-07-16 09:59:22,707 INFO    MainThread:782479 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-16 09:59:22,884 INFO    MainThread:782479 [wandb_init.py:init():529] starting run threads in backend
+2021-07-16 09:59:24,052 INFO    MainThread:782479 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-16 09:59:24,053 INFO    MainThread:782479 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-16 09:59:24,054 INFO    MainThread:782479 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-16 09:59:24,055 INFO    MainThread:782479 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-16 09:59:24,056 INFO    MainThread:782479 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-16 09:59:24,061 INFO    MainThread:782479 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul16_09-59-13_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 15000, 'save_total_limit': 20, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 6000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
+2021-07-16 09:59:24,063 INFO    MainThread:782479 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-16 09:59:24,065 INFO    MainThread:782479 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 4000}