Saving weights and logs at step 5

events.out.tfevents.1626298536.t1v-n-f5c06ea1-w-0.581357.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6140a1c229a49ca5487ff7762776759d5ea43ddad0d97c1e0b7389f25cae2ad
+size 40

events.out.tfevents.1626298784.t1v-n-f5c06ea1-w-0.583081.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a72b4b117c623ae62a396471264fcbea77d5fb762251237be0b92401d939a95
+size 40

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba765ca93df3c067ee3d3ec56bc471ecc71072620bbbdc50509b516efdbf60c4
+oid sha256:a07e8333f5822e57ebf3a1c59c4f6c6d734efaf0017732c6c8254869a4894524
 size 510090043

opt_state.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:328aaa347b117dca77396a9d1fbc5f412fc5e22d9f5bf9e8b2eb8b89202f97d9
+size 1530270545

run.sh

@@ -18,7 +18,7 @@ python ./run_mlm_flax.py \
     --eval_steps="500" \
     --num_train_epochs="5" \
     --preprocessing_num_workers="96" \
-    --save_steps="500" \
+    --save_steps="5" \
     --learning_rate="5e-5" \
     --per_device_train_batch_size="2" \
     --per_device_eval_batch_size="2" \

run_mlm_flax.py

@@ -339,6 +339,17 @@ def save_model_checkpoint(model, save_dir, state, with_opt:bool=True, push_to_hu
             json.dump({"step": state.step.item()}, f)
     logger.info("checkpoint saved")
 
+def rotate_checkpoints(ckpt_dir:str, save_total_limit:int):
+    "Removes older checkpoints so that `save_total_limit` checkpoints are kept"
+    # TODO: what to remove is decided using step number only, we might want to improve that
+    ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")]
+    # sort checkpoints by step
+    ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1]))
+    ckpts_to_delete = ckpts_sorted[:-save_total_limit]
+    for ckpt in ckpts_to_delete:
+        logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})")
+        shutil.rmtree(ckpt)
+
 
 
 if __name__ == "__main__":

training_state.json

@@ -0,0 +1 @@
+{"step": 501}

wandb/debug-internal.log

@@ -1 +1 @@
-run-20210714_212350-6qjir2uu/logs/debug-internal.log
+run-20210714_213944-3j6d3fy2/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20210714_212350-6qjir2uu/logs/debug.log
+run-20210714_213944-3j6d3fy2/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20210714_212350-6qjir2uu
+run-20210714_213944-3j6d3fy2

wandb/run-20210714_212350-6qjir2uu/files/config.yaml

@@ -13,6 +13,9 @@ _wandb:
       1:
       - 3
       - 11
+      2:
+      - 3
+      - 11
       4: 3.8.10
       5: 0.10.33
       6: 4.9.0.dev0

wandb/run-20210714_212350-6qjir2uu/files/output.log

@@ -171,3 +171,15 @@ Evaluating ...:   0%|
 
 [21:32:05] - INFO - huggingface_hub.repository - git version 2.25.1██████████████████████████████████████████████████████████████████████████████████| 31/31 [00:21<00:00,  9.98it/s]
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
+[21:32:05] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[21:32:35] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (2/2), 510 MB | 31 MB/s, done.
+tcmalloc: large alloc 1354776576 bytes == 0x304b28000 @  0x7fd74488d680 0x7fd7448adbdd 0x7fd478ac320d 0x7fd478ad1340 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478accbd3 0x7fd478acd1fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56aadf 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161
+tcmalloc: large alloc 2715181056 bytes == 0x35572c000 @  0x7fd74488d680 0x7fd7448adbdd 0x7fd478ac320d 0x7fd478ad1340 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478ad0e87 0x7fd478accbd3 0x7fd478acd1fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56aadf 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161 0x67e1df
+tcmalloc: large alloc 1530273792 bytes == 0x2ae462000 @  0x7fd74488d680 0x7fd7448ae824 0x5f7b11 0x7fd478accc6f 0x7fd478acd1fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56aadf 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fd7446a20b3 0x5f96de
+[21:32:57] - INFO - __main__ - checkpoint saved
+Training...:  40%|███████████████████████████████████████████████████▏                                                                            | 500/1250 [08:46<13:09,  1.05s/it]
+Step... (500 | Loss: 10.108721733093262, Acc: 0.043713752180337906):   0%|                                                                                     | 0/5 [08:51<?, ?it/s]
+Traceback (most recent call last):
+  File "./run_mlm_flax.py", line 853, in <module>
+    rotate_checkpoints(training_args.output_dir, training_args.save_total_limit)
+NameError: name 'rotate_checkpoints' is not defined

wandb/run-20210714_212350-6qjir2uu/logs/debug-internal.log

@@ -289,3 +289,118 @@
 2021-07-14 21:32:07,771 INFO    Thread-8  :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
 2021-07-14 21:32:12,254 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-14 21:32:12,255 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:32:21,959 DEBUG   SenderThread:580021 [sender.py:send():179] send: stats
+2021-07-14 21:32:27,390 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:32:27,391 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:32:37,785 INFO    Thread-8  :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
+2021-07-14 21:32:41,786 INFO    Thread-8  :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
+2021-07-14 21:32:42,870 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:32:42,871 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:32:43,788 INFO    Thread-8  :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
+2021-07-14 21:32:52,027 DEBUG   SenderThread:580021 [sender.py:send():179] send: stats
+2021-07-14 21:32:59,795 INFO    Thread-8  :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
+2021-07-14 21:32:59,895 DEBUG   SenderThread:580021 [sender.py:send():179] send: telemetry
+2021-07-14 21:32:59,896 DEBUG   SenderThread:580021 [sender.py:send():179] send: exit
+2021-07-14 21:32:59,896 INFO    SenderThread:580021 [sender.py:send_exit():287] handling exit code: 1
+2021-07-14 21:32:59,896 INFO    SenderThread:580021 [sender.py:send_exit():295] send defer
+2021-07-14 21:32:59,896 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:32:59,897 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:32:59,897 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-14 21:32:59,897 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:32:59,897 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:32:59,897 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-14 21:32:59,897 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 1
+2021-07-14 21:32:59,897 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:32:59,898 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-14 21:32:59,944 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:32:59,944 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-14 21:32:59,944 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 2
+2021-07-14 21:32:59,944 DEBUG   SenderThread:580021 [sender.py:send():179] send: stats
+2021-07-14 21:32:59,945 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:32:59,945 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-14 21:32:59,945 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:32:59,945 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-14 21:32:59,945 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 3
+2021-07-14 21:32:59,945 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:32:59,946 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-14 21:32:59,946 DEBUG   SenderThread:580021 [sender.py:send():179] send: summary
+2021-07-14 21:32:59,946 INFO    SenderThread:580021 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-14 21:32:59,946 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:32:59,946 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-14 21:32:59,946 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 4
+2021-07-14 21:32:59,947 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:32:59,947 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-14 21:32:59,947 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:32:59,947 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-14 21:32:59,999 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:00,122 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 5
+2021-07-14 21:33:00,122 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:00,122 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:33:00,122 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-14 21:33:00,123 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:33:00,123 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-14 21:33:00,123 INFO    SenderThread:580021 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-14 21:33:00,224 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:00,795 INFO    Thread-8  :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/config.yaml
+2021-07-14 21:33:00,796 INFO    SenderThread:580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
+2021-07-14 21:33:00,796 INFO    SenderThread:580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json
+2021-07-14 21:33:00,796 INFO    SenderThread:580021 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files
+2021-07-14 21:33:00,796 INFO    SenderThread:580021 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/requirements.txt requirements.txt
+2021-07-14 21:33:00,796 INFO    SenderThread:580021 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log output.log
+2021-07-14 21:33:00,797 INFO    SenderThread:580021 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-metadata.json wandb-metadata.json
+2021-07-14 21:33:00,797 INFO    SenderThread:580021 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/config.yaml config.yaml
+2021-07-14 21:33:00,797 INFO    SenderThread:580021 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json wandb-summary.json
+2021-07-14 21:33:00,797 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 6
+2021-07-14 21:33:00,797 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:00,801 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:33:00,801 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-14 21:33:00,804 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:33:00,805 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-14 21:33:00,805 INFO    SenderThread:580021 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-14 21:33:00,903 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:00,903 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,005 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,006 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,108 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,108 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,210 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,210 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,236 INFO    Thread-14 :580021 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/config.yaml
+2021-07-14 21:33:01,247 INFO    Thread-12 :580021 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/requirements.txt
+2021-07-14 21:33:01,250 INFO    Thread-15 :580021 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json
+2021-07-14 21:33:01,286 INFO    Thread-13 :580021 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
+2021-07-14 21:33:01,312 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,312 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,414 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,414 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,486 INFO    Thread-7  :580021 [sender.py:transition_state():308] send defer: 7
+2021-07-14 21:33:01,487 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:33:01,487 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-14 21:33:01,487 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:33:01,487 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-14 21:33:01,516 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,759 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 8
+2021-07-14 21:33:01,759 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,759 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:33:01,760 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-14 21:33:01,760 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:33:01,760 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-14 21:33:01,760 INFO    SenderThread:580021 [sender.py:transition_state():308] send defer: 9
+2021-07-14 21:33:01,760 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:33:01,760 DEBUG   SenderThread:580021 [sender.py:send():179] send: final
+2021-07-14 21:33:01,761 INFO    HandlerThread:580021 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-14 21:33:01,761 DEBUG   SenderThread:580021 [sender.py:send():179] send: footer
+2021-07-14 21:33:01,761 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:33:01,761 INFO    SenderThread:580021 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-14 21:33:01,861 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:33:01,861 DEBUG   SenderThread:580021 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:33:01,861 INFO    SenderThread:580021 [file_pusher.py:join():182] waiting for file pusher
+2021-07-14 21:33:01,863 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-14 21:33:01,864 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-14 21:33:01,865 DEBUG   HandlerThread:580021 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-14 21:33:01,866 INFO    HandlerThread:580021 [handler.py:finish():638] shutting down handler
+2021-07-14 21:33:02,761 INFO    WriterThread:580021 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/run-6qjir2uu.wandb
+2021-07-14 21:33:02,862 INFO    SenderThread:580021 [sender.py:finish():945] shutting down sender
+2021-07-14 21:33:02,862 INFO    SenderThread:580021 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-14 21:33:02,862 INFO    SenderThread:580021 [file_pusher.py:join():182] waiting for file pusher
+2021-07-14 21:33:02,864 INFO    MainThread:580021 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210714_212350-6qjir2uu/logs/debug.log

@@ -23,3 +23,99 @@ config: {}
 2021-07-14 21:23:52,724 INFO    MainThread:578764 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-23-42_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
 2021-07-14 21:23:52,726 INFO    MainThread:578764 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True}
 2021-07-14 21:23:52,727 INFO    MainThread:578764 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
+2021-07-14 21:32:57,866 INFO    MainThread:578764 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1
+2021-07-14 21:32:57,867 INFO    MainThread:578764 [wandb_run.py:_restore():1565] restore
+2021-07-14 21:32:59,898 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1442
+  total_bytes: 1442
+}
+
+2021-07-14 21:33:00,123 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1442
+  total_bytes: 1442
+}
+
+2021-07-14 21:33:00,802 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 4
+}
+pusher_stats {
+  uploaded_bytes: 1442
+  total_bytes: 12361
+}
+
+2021-07-14 21:33:00,904 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 1442
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,006 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,109 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,210 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,312 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,414 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,760 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:01,862 INFO    MainThread:578764 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 12603
+  total_bytes: 12603
+}
+
+2021-07-14 21:33:03,150 INFO    MainThread:578764 [wandb_run.py:_show_summary():1870] rendering summary
+2021-07-14 21:33:03,150 INFO    MainThread:578764 [wandb_run.py:_show_history():1908] rendering history
+2021-07-14 21:33:03,150 INFO    MainThread:578764 [wandb_run.py:_show_files():1937] logging synced files

wandb/run-20210714_213537-20l16od8/files/config.yaml

@@ -0,0 +1,307 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 3
+      - 11
+      2:
+      - 3
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.98
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: null
+dataset_name:
+  desc: null
+  value: null
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 4
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 5.0e-05
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul14_21-35-30_t1v-n-f5c06ea1-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 250
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_eval_samples:
+  desc: null
+  value: 500
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 4096
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: big_bird
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 5.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 2
+per_device_train_batch_size:
+  desc: null
+  value: 2
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: 96
+push_to_hub:
+  desc: null
+  value: true
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: ./
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_optimizer:
+  desc: null
+  value: true
+save_steps:
+  desc: null
+  value: 500
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: 5
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 5000
+weight_decay:
+  desc: null
+  value: 0.0095

wandb/run-20210714_213537-20l16od8/files/output.log

@@ -0,0 +1,15 @@
+[21:35:52] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
+/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3132: UserWarning: Explicitly requested dtype <class 'jax._src.numpy.lax_numpy.int64'> requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more.
+  lax._check_user_dtype_supported(dtype, "zeros")
+[21:35:52] - INFO - __main__ - RESTORING CHECKPOINT FROM ./...
+tcmalloc: large alloc 1530273792 bytes == 0xd8c8e000 @  0x7f41b5469680 0x7f41b548a824 0x5f7b11 0x648631 0x5c38e6 0x4f30e6 0x64ee88 0x505653 0x56acb6 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7f41b527e0b3 0x5f96de
+restoring state of multisteps optimizer
+[21:35:55] - INFO - __main__ - checkpoint restored
+Traceback (most recent call last):
+  File "./run_mlm_flax.py", line 712, in <module>
+    state = restore_model_checkpoint(training_args.resume_from_checkpoint, state)
+  File "./run_mlm_flax.py", line 314, in restore_model_checkpoint
+    inner_opt_state = reinstantiate_states(opt_state.inner_opt_state)
+  File "./run_mlm_flax.py", line 294, in reinstantiate_states
+    cls = getattr(optax, type(state).__name__)
+AttributeError: module 'optax' has no attribute 'list'

wandb/run-20210714_213537-20l16od8/files/requirements.txt

@@ -0,0 +1,94 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==21.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+charset-normalizer==2.0.1
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.7.0
+gast==0.4.0
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+google-pasta==0.2.0
+grpcio==1.34.1
+h5py==3.1.0
+huggingface-hub==0.0.12
+idna==3.2
+install==1.3.4
+jax==0.2.17
+jaxlib==0.1.68
+joblib==1.0.1
+keras-nightly==2.5.0.dev2021032900
+keras-preprocessing==1.1.2
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.19.5
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+pathtools==0.1.2
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+protobuf==3.17.3
+psutil==5.8.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.15.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.5.0
+tensorflow==2.5.0
+termcolor==1.1.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+transformers==4.9.0.dev0
+typing-extensions==3.7.4.3
+urllib3==1.26.6
+wandb==0.10.33
+werkzeug==2.0.1
+wheel==0.36.2
+wrapt==1.12.1
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210714_213537-20l16od8/files/wandb-metadata.json

@@ -0,0 +1,48 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-14T21:35:39.374289",
+    "startedAt": "2021-07-14T21:35:37.412426",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--push_to_hub",
+        "--output_dir=./",
+        "--model_type=big_bird",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--max_seq_length=4096",
+        "--weight_decay=0.0095",
+        "--warmup_steps=5000",
+        "--overwrite_output_dir",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--logging_steps=250",
+        "--eval_steps=500",
+        "--num_train_epochs=5",
+        "--preprocessing_num_workers=96",
+        "--save_steps=500",
+        "--learning_rate=5e-5",
+        "--per_device_train_batch_size=2",
+        "--per_device_eval_batch_size=2",
+        "--save_total_limit=5",
+        "--max_eval_samples=500",
+        "--overwrite_cache",
+        "False",
+        "--gradient_accumulation_steps=4",
+        "--resume_from_checkpoint=./"
+    ],
+    "state": "running",
+    "program": "./run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/pino-roberta-base",
+        "commit": "5551a58bbb1262f3d17d25171ffa2da47567c22a"
+    },
+    "email": null,
+    "root": "/home/dat/pino-roberta-base",
+    "host": "t1v-n-f5c06ea1-w-0",
+    "username": "dat",
+    "executable": "/home/dat/pino/bin/python"
+}

wandb/run-20210714_213537-20l16od8/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210714_213537-20l16od8/logs/debug-internal.log

@@ -0,0 +1,141 @@
+2021-07-14 21:35:38,058 INFO    MainThread:582612 [internal.py:wandb_internal():88] W&B internal server running at pid: 582612, started at: 2021-07-14 21:35:38.058075
+2021-07-14 21:35:38,060 INFO    WriterThread:582612 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/run-20l16od8.wandb
+2021-07-14 21:35:38,060 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: check_version
+2021-07-14 21:35:38,061 DEBUG   SenderThread:582612 [sender.py:send():179] send: header
+2021-07-14 21:35:38,061 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: check_version
+2021-07-14 21:35:38,099 DEBUG   SenderThread:582612 [sender.py:send():179] send: run
+2021-07-14 21:35:38,275 INFO    SenderThread:582612 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files
+2021-07-14 21:35:38,275 INFO    SenderThread:582612 [sender.py:_start_run_threads():716] run started: 20l16od8 with start time 1626298537
+2021-07-14 21:35:38,275 DEBUG   SenderThread:582612 [sender.py:send():179] send: summary
+2021-07-14 21:35:38,275 INFO    SenderThread:582612 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-14 21:35:38,276 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: run_start
+2021-07-14 21:35:39,277 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/wandb-summary.json
+2021-07-14 21:35:39,374 DEBUG   HandlerThread:582612 [meta.py:__init__():39] meta init
+2021-07-14 21:35:39,374 DEBUG   HandlerThread:582612 [meta.py:__init__():53] meta init done
+2021-07-14 21:35:39,374 DEBUG   HandlerThread:582612 [meta.py:probe():210] probe
+2021-07-14 21:35:39,375 DEBUG   HandlerThread:582612 [meta.py:_setup_git():200] setup git
+2021-07-14 21:35:39,404 DEBUG   HandlerThread:582612 [meta.py:_setup_git():207] setup git done
+2021-07-14 21:35:39,405 DEBUG   HandlerThread:582612 [meta.py:_save_pip():57] save pip
+2021-07-14 21:35:39,405 DEBUG   HandlerThread:582612 [meta.py:_save_pip():71] save pip done
+2021-07-14 21:35:39,405 DEBUG   HandlerThread:582612 [meta.py:probe():252] probe done
+2021-07-14 21:35:39,408 DEBUG   SenderThread:582612 [sender.py:send():179] send: files
+2021-07-14 21:35:39,408 INFO    SenderThread:582612 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
+2021-07-14 21:35:39,414 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:35:39,415 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:35:39,542 DEBUG   SenderThread:582612 [sender.py:send():179] send: config
+2021-07-14 21:35:39,542 DEBUG   SenderThread:582612 [sender.py:send():179] send: config
+2021-07-14 21:35:39,542 DEBUG   SenderThread:582612 [sender.py:send():179] send: config
+2021-07-14 21:35:39,845 INFO    Thread-11 :582612 [upload_job.py:push():137] Uploaded file /tmp/tmp81cfdra0wandb/3f4lkdmt-wandb-metadata.json
+2021-07-14 21:35:40,276 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/requirements.txt
+2021-07-14 21:35:40,276 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/wandb-metadata.json
+2021-07-14 21:35:40,276 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log
+2021-07-14 21:35:54,281 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log
+2021-07-14 21:35:54,573 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:35:54,574 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:35:56,282 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log
+2021-07-14 21:35:57,283 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log
+2021-07-14 21:35:58,079 DEBUG   SenderThread:582612 [sender.py:send():179] send: telemetry
+2021-07-14 21:35:58,080 DEBUG   SenderThread:582612 [sender.py:send():179] send: exit
+2021-07-14 21:35:58,080 INFO    SenderThread:582612 [sender.py:send_exit():287] handling exit code: 1
+2021-07-14 21:35:58,080 INFO    SenderThread:582612 [sender.py:send_exit():295] send defer
+2021-07-14 21:35:58,080 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:58,080 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:58,081 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:58,081 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-14 21:35:58,081 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:58,081 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-14 21:35:58,081 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 1
+2021-07-14 21:35:58,081 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:58,081 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-14 21:35:58,128 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:58,128 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-14 21:35:58,128 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 2
+2021-07-14 21:35:58,128 DEBUG   SenderThread:582612 [sender.py:send():179] send: stats
+2021-07-14 21:35:58,128 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:58,129 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-14 21:35:58,129 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:58,129 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-14 21:35:58,129 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 3
+2021-07-14 21:35:58,129 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:58,129 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-14 21:35:58,129 DEBUG   SenderThread:582612 [sender.py:send():179] send: summary
+2021-07-14 21:35:58,130 INFO    SenderThread:582612 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-14 21:35:58,130 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:58,130 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-14 21:35:58,130 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 4
+2021-07-14 21:35:58,130 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:58,130 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-14 21:35:58,130 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:58,130 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-14 21:35:58,183 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:58,299 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 5
+2021-07-14 21:35:58,299 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:58,300 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log
+2021-07-14 21:35:58,300 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/wandb-summary.json
+2021-07-14 21:35:58,300 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:58,300 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-14 21:35:58,300 INFO    Thread-8  :582612 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/config.yaml
+2021-07-14 21:35:58,301 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:58,301 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-14 21:35:58,301 INFO    SenderThread:582612 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-14 21:35:58,402 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,301 INFO    SenderThread:582612 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files
+2021-07-14 21:35:59,302 INFO    SenderThread:582612 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/requirements.txt requirements.txt
+2021-07-14 21:35:59,302 INFO    SenderThread:582612 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log output.log
+2021-07-14 21:35:59,302 INFO    SenderThread:582612 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/wandb-metadata.json wandb-metadata.json
+2021-07-14 21:35:59,302 INFO    SenderThread:582612 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/config.yaml config.yaml
+2021-07-14 21:35:59,302 INFO    SenderThread:582612 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/wandb-summary.json wandb-summary.json
+2021-07-14 21:35:59,302 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 6
+2021-07-14 21:35:59,303 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,303 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:59,303 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-14 21:35:59,307 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:59,307 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-14 21:35:59,307 INFO    SenderThread:582612 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-14 21:35:59,405 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,405 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,507 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,507 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,609 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,609 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,711 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,711 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,738 INFO    Thread-15 :582612 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/wandb-summary.json
+2021-07-14 21:35:59,759 INFO    Thread-13 :582612 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/output.log
+2021-07-14 21:35:59,764 INFO    Thread-14 :582612 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/config.yaml
+2021-07-14 21:35:59,779 INFO    Thread-12 :582612 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/files/requirements.txt
+2021-07-14 21:35:59,813 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,813 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,915 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:35:59,915 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:35:59,979 INFO    Thread-7  :582612 [sender.py:transition_state():308] send defer: 7
+2021-07-14 21:35:59,980 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:35:59,980 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-14 21:35:59,980 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:35:59,980 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-14 21:36:00,016 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:36:00,132 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 8
+2021-07-14 21:36:00,132 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:36:00,133 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:36:00,133 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-14 21:36:00,133 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:36:00,133 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-14 21:36:00,133 INFO    SenderThread:582612 [sender.py:transition_state():308] send defer: 9
+2021-07-14 21:36:00,134 DEBUG   SenderThread:582612 [sender.py:send():179] send: final
+2021-07-14 21:36:00,134 DEBUG   SenderThread:582612 [sender.py:send():179] send: footer
+2021-07-14 21:36:00,134 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: defer
+2021-07-14 21:36:00,134 INFO    HandlerThread:582612 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-14 21:36:00,134 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: defer
+2021-07-14 21:36:00,134 INFO    SenderThread:582612 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-14 21:36:00,234 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-14 21:36:00,235 DEBUG   SenderThread:582612 [sender.py:send_request():193] send_request: poll_exit
+2021-07-14 21:36:00,235 INFO    SenderThread:582612 [file_pusher.py:join():182] waiting for file pusher
+2021-07-14 21:36:00,236 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-14 21:36:00,237 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-14 21:36:00,237 DEBUG   HandlerThread:582612 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-14 21:36:00,237 INFO    HandlerThread:582612 [handler.py:finish():638] shutting down handler
+2021-07-14 21:36:01,134 INFO    WriterThread:582612 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/run-20l16od8.wandb
+2021-07-14 21:36:01,235 INFO    SenderThread:582612 [sender.py:finish():945] shutting down sender
+2021-07-14 21:36:01,235 INFO    SenderThread:582612 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-14 21:36:01,235 INFO    SenderThread:582612 [file_pusher.py:join():182] waiting for file pusher
+2021-07-14 21:36:01,237 INFO    MainThread:582612 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210714_213537-20l16od8/logs/debug.log

@@ -0,0 +1,119 @@
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_setup.py:_flush():69] setting env: {}
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/logs/debug.log
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_213537-20l16od8/logs/debug-internal.log
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_init.py:init():370] calling init triggers
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [wandb_init.py:init():419] starting backend
+2021-07-14 21:35:37,414 INFO    MainThread:581357 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-14 21:35:37,461 INFO    MainThread:581357 [backend.py:ensure_launched():135] starting backend process...
+2021-07-14 21:35:37,506 INFO    MainThread:581357 [backend.py:ensure_launched():139] started backend process with pid: 582612
+2021-07-14 21:35:37,508 INFO    MainThread:581357 [wandb_init.py:init():424] backend started and connected
+2021-07-14 21:35:37,511 INFO    MainThread:581357 [wandb_init.py:init():472] updated telemetry
+2021-07-14 21:35:37,512 INFO    MainThread:581357 [wandb_init.py:init():491] communicating current version
+2021-07-14 21:35:38,098 INFO    MainThread:581357 [wandb_init.py:init():496] got version response 
+2021-07-14 21:35:38,098 INFO    MainThread:581357 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-14 21:35:38,275 INFO    MainThread:581357 [wandb_init.py:init():529] starting run threads in backend
+2021-07-14 21:35:39,411 INFO    MainThread:581357 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-14 21:35:39,412 INFO    MainThread:581357 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-14 21:35:39,412 INFO    MainThread:581357 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-14 21:35:39,414 INFO    MainThread:581357 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-14 21:35:39,414 INFO    MainThread:581357 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-14 21:35:39,420 INFO    MainThread:581357 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-35-30_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-14 21:35:39,422 INFO    MainThread:581357 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True}
+2021-07-14 21:35:39,424 INFO    MainThread:581357 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
+2021-07-14 21:35:55,601 INFO    MainThread:581357 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1
+2021-07-14 21:35:55,602 INFO    MainThread:581357 [wandb_run.py:_restore():1565] restore
+2021-07-14 21:35:58,081 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1481
+  total_bytes: 1481
+}
+
+2021-07-14 21:35:58,301 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1481
+  total_bytes: 1481
+}
+
+2021-07-14 21:35:59,303 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 4
+}
+pusher_stats {
+  uploaded_bytes: 1481
+  total_bytes: 9251
+}
+
+2021-07-14 21:35:59,406 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 1481
+  total_bytes: 9253
+}
+
+2021-07-14 21:35:59,508 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:35:59,610 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:35:59,712 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:35:59,814 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:35:59,915 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:36:00,133 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:36:00,235 INFO    MainThread:581357 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 9253
+  total_bytes: 9253
+}
+
+2021-07-14 21:36:01,503 INFO    MainThread:581357 [wandb_run.py:_show_files():1937] logging synced files

wandb/run-20210714_213944-3j6d3fy2/files/config.yaml

@@ -0,0 +1,304 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 3
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.98
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: ./
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: null
+dataset_name:
+  desc: null
+  value: null
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 500
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 4
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 5.0e-05
+length_column_name:
+  desc: null
+  value: length
+line_by_line:
+  desc: null
+  value: false
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Jul14_21-39-37_t1v-n-f5c06ea1-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 250
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_eval_samples:
+  desc: null
+  value: 500
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_seq_length:
+  desc: null
+  value: 4096
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: null
+mlm_probability:
+  desc: null
+  value: 0.15
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: big_bird
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 5.0
+output_dir:
+  desc: null
+  value: ./
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_to_max_length:
+  desc: null
+  value: false
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 2
+per_device_train_batch_size:
+  desc: null
+  value: 2
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: 96
+push_to_hub:
+  desc: null
+  value: true
+push_to_hub_model_id:
+  desc: null
+  value: ''
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: ./
+save_on_each_node:
+  desc: null
+  value: false
+save_optimizer:
+  desc: null
+  value: true
+save_steps:
+  desc: null
+  value: 5
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: 5
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+tokenizer_name:
+  desc: null
+  value: ./
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_ref_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_ref_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 5
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 5000
+weight_decay:
+  desc: null
+  value: 0.0095

wandb/run-20210714_213944-3j6d3fy2/files/output.log

@@ -0,0 +1,12 @@
+[21:39:59] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
+/home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3132: UserWarning: Explicitly requested dtype <class 'jax._src.numpy.lax_numpy.int64'> requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more.
+  lax._check_user_dtype_supported(dtype, "zeros")
+/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  warnings.warn(
+/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  warnings.warn(
+Epoch ... (1/5):   0%|                                                                                                                                         | 0/5 [00:00<?, ?it/s][21:40:00] - INFO - __main__ - Skipping to epoch 0 step 0
+
+
+[21:41:37] - INFO - huggingface_hub.repository - git version 2.25.1                                                                               | 5/1250 [01:27<2:38:49,  7.65s/it]
+git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)

wandb/run-20210714_213944-3j6d3fy2/files/requirements.txt

@@ -0,0 +1,94 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==21.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+charset-normalizer==2.0.1
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cycler==0.10.0
+datasets==1.9.1.dev0
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+filelock==3.0.12
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.7.0
+gast==0.4.0
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+google-pasta==0.2.0
+grpcio==1.34.1
+h5py==3.1.0
+huggingface-hub==0.0.12
+idna==3.2
+install==1.3.4
+jax==0.2.17
+jaxlib==0.1.68
+joblib==1.0.1
+keras-nightly==2.5.0.dev2021032900
+keras-preprocessing==1.1.2
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+matplotlib==3.4.2
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+numpy==1.19.5
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.3.0
+pathtools==0.1.2
+pillow==8.3.1
+pip==20.0.2
+pkg-resources==0.0.0
+promise==2.3
+protobuf==3.17.3
+psutil==5.8.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.7.6
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+sentry-sdk==1.3.0
+setuptools==44.0.0
+shortuuid==1.0.1
+six==1.15.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.5.0
+tensorflow==2.5.0
+termcolor==1.1.0
+tokenizers==0.10.3
+toolz==0.11.1
+tqdm==4.61.2
+transformers==4.9.0.dev0
+typing-extensions==3.7.4.3
+urllib3==1.26.6
+wandb==0.10.33
+werkzeug==2.0.1
+wheel==0.36.2
+wrapt==1.12.1
+xxhash==2.0.2
+yarl==1.6.3

wandb/run-20210714_213944-3j6d3fy2/files/wandb-metadata.json

@@ -0,0 +1,47 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-14T21:39:46.918417",
+    "startedAt": "2021-07-14T21:39:44.931675",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--push_to_hub",
+        "--output_dir=./",
+        "--model_type=big_bird",
+        "--config_name=./",
+        "--tokenizer_name=./",
+        "--max_seq_length=4096",
+        "--weight_decay=0.0095",
+        "--warmup_steps=5000",
+        "--overwrite_output_dir",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--logging_steps=250",
+        "--eval_steps=500",
+        "--num_train_epochs=5",
+        "--preprocessing_num_workers=96",
+        "--save_steps=5",
+        "--learning_rate=5e-5",
+        "--per_device_train_batch_size=2",
+        "--per_device_eval_batch_size=2",
+        "--save_total_limit=5",
+        "--max_eval_samples=500",
+        "--overwrite_cache",
+        "False",
+        "--gradient_accumulation_steps=4"
+    ],
+    "state": "running",
+    "program": "./run_mlm_flax.py",
+    "codePath": "run_mlm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/pino-roberta-base",
+        "commit": "5551a58bbb1262f3d17d25171ffa2da47567c22a"
+    },
+    "email": null,
+    "root": "/home/dat/pino-roberta-base",
+    "host": "t1v-n-f5c06ea1-w-0",
+    "username": "dat",
+    "executable": "/home/dat/pino/bin/python"
+}

wandb/run-20210714_213944-3j6d3fy2/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210714_213944-3j6d3fy2/logs/debug-internal.log

@@ -0,0 +1,57 @@
+2021-07-14 21:39:45,622 INFO    MainThread:584332 [internal.py:wandb_internal():88] W&B internal server running at pid: 584332, started at: 2021-07-14 21:39:45.622063
+2021-07-14 21:39:45,624 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: check_version
+2021-07-14 21:39:45,624 INFO    WriterThread:584332 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/run-3j6d3fy2.wandb
+2021-07-14 21:39:45,625 DEBUG   SenderThread:584332 [sender.py:send():179] send: header
+2021-07-14 21:39:45,625 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: check_version
+2021-07-14 21:39:45,661 DEBUG   SenderThread:584332 [sender.py:send():179] send: run
+2021-07-14 21:39:45,830 INFO    SenderThread:584332 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files
+2021-07-14 21:39:45,830 INFO    SenderThread:584332 [sender.py:_start_run_threads():716] run started: 3j6d3fy2 with start time 1626298785
+2021-07-14 21:39:45,830 DEBUG   SenderThread:584332 [sender.py:send():179] send: summary
+2021-07-14 21:39:45,830 INFO    SenderThread:584332 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-14 21:39:45,831 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: run_start
+2021-07-14 21:39:46,834 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/wandb-summary.json
+2021-07-14 21:39:46,918 DEBUG   HandlerThread:584332 [meta.py:__init__():39] meta init
+2021-07-14 21:39:46,918 DEBUG   HandlerThread:584332 [meta.py:__init__():53] meta init done
+2021-07-14 21:39:46,918 DEBUG   HandlerThread:584332 [meta.py:probe():210] probe
+2021-07-14 21:39:46,919 DEBUG   HandlerThread:584332 [meta.py:_setup_git():200] setup git
+2021-07-14 21:39:46,949 DEBUG   HandlerThread:584332 [meta.py:_setup_git():207] setup git done
+2021-07-14 21:39:46,949 DEBUG   HandlerThread:584332 [meta.py:_save_pip():57] save pip
+2021-07-14 21:39:46,949 DEBUG   HandlerThread:584332 [meta.py:_save_pip():71] save pip done
+2021-07-14 21:39:46,949 DEBUG   HandlerThread:584332 [meta.py:probe():252] probe done
+2021-07-14 21:39:46,952 DEBUG   SenderThread:584332 [sender.py:send():179] send: files
+2021-07-14 21:39:46,952 INFO    SenderThread:584332 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
+2021-07-14 21:39:46,958 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:39:46,959 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:39:47,087 DEBUG   SenderThread:584332 [sender.py:send():179] send: config
+2021-07-14 21:39:47,088 DEBUG   SenderThread:584332 [sender.py:send():179] send: config
+2021-07-14 21:39:47,088 DEBUG   SenderThread:584332 [sender.py:send():179] send: config
+2021-07-14 21:39:47,442 INFO    Thread-11 :584332 [upload_job.py:push():137] Uploaded file /tmp/tmp4jyyzi24wandb/1yoeh974-wandb-metadata.json
+2021-07-14 21:39:47,832 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/wandb-metadata.json
+2021-07-14 21:39:47,832 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/output.log
+2021-07-14 21:39:47,832 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/requirements.txt
+2021-07-14 21:40:01,837 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/output.log
+2021-07-14 21:40:02,089 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:40:02,090 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:40:07,839 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/output.log
+2021-07-14 21:40:14,997 DEBUG   SenderThread:584332 [sender.py:send():179] send: stats
+2021-07-14 21:40:16,842 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/config.yaml
+2021-07-14 21:40:17,237 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:40:17,237 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:40:32,378 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:40:32,378 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:40:45,080 DEBUG   SenderThread:584332 [sender.py:send():179] send: stats
+2021-07-14 21:40:47,511 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:40:47,511 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:41:02,643 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:41:02,643 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:41:15,151 DEBUG   SenderThread:584332 [sender.py:send():179] send: stats
+2021-07-14 21:41:17,776 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:41:17,776 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:41:32,912 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:41:32,912 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status
+2021-07-14 21:41:33,872 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/output.log
+2021-07-14 21:41:35,873 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/output.log
+2021-07-14 21:41:39,875 INFO    Thread-8  :584332 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/files/output.log
+2021-07-14 21:41:45,230 DEBUG   SenderThread:584332 [sender.py:send():179] send: stats
+2021-07-14 21:41:48,051 DEBUG   HandlerThread:584332 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-14 21:41:48,052 DEBUG   SenderThread:584332 [sender.py:send_request():193] send_request: stop_status

wandb/run-20210714_213944-3j6d3fy2/logs/debug.log

@@ -0,0 +1,25 @@
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_setup.py:_flush():69] setting env: {}
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/logs/debug.log
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_213944-3j6d3fy2/logs/debug-internal.log
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_init.py:init():370] calling init triggers
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [wandb_init.py:init():419] starting backend
+2021-07-14 21:39:44,933 INFO    MainThread:583081 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-14 21:39:44,979 INFO    MainThread:583081 [backend.py:ensure_launched():135] starting backend process...
+2021-07-14 21:39:45,023 INFO    MainThread:583081 [backend.py:ensure_launched():139] started backend process with pid: 584332
+2021-07-14 21:39:45,025 INFO    MainThread:583081 [wandb_init.py:init():424] backend started and connected
+2021-07-14 21:39:45,028 INFO    MainThread:583081 [wandb_init.py:init():472] updated telemetry
+2021-07-14 21:39:45,029 INFO    MainThread:583081 [wandb_init.py:init():491] communicating current version
+2021-07-14 21:39:45,659 INFO    MainThread:583081 [wandb_init.py:init():496] got version response 
+2021-07-14 21:39:45,659 INFO    MainThread:583081 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-14 21:39:45,830 INFO    MainThread:583081 [wandb_init.py:init():529] starting run threads in backend
+2021-07-14 21:39:46,956 INFO    MainThread:583081 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-14 21:39:46,956 INFO    MainThread:583081 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-14 21:39:46,957 INFO    MainThread:583081 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-14 21:39:46,959 INFO    MainThread:583081 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-14 21:39:46,959 INFO    MainThread:583081 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-14 21:39:46,965 INFO    MainThread:583081 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-39-37_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 5, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
+2021-07-14 21:39:46,971 INFO    MainThread:583081 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True}
+2021-07-14 21:39:46,972 INFO    MainThread:583081 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}