dat commited on
Commit
5551a58
1 Parent(s): f291f93

Saving weights and logs at step 500

Browse files
events.out.tfevents.1626297829.t1v-n-f5c06ea1-w-0.578764.3.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14dfc06e6c8c2d64641ac5905c79546cccc7cc70c30040cbc4dcf5c4215f8151
3
+ size 73705
flax_model.msgpack CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:422812fccdda54c02543ac5e994b33b54e510e0474439fbe9360d5190787d38e
3
  size 510090043
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba765ca93df3c067ee3d3ec56bc471ecc71072620bbbdc50509b516efdbf60c4
3
  size 510090043
opt_state.msgpack ADDED
File without changes
run.sh CHANGED
@@ -16,10 +16,10 @@ python ./run_mlm_flax.py \
16
  --adam_beta2="0.98" \
17
  --logging_steps="250" \
18
  --eval_steps="500" \
19
- --num_train_epochs="3" \
20
  --preprocessing_num_workers="96" \
21
- --save_steps="1250" \
22
- --learning_rate="1e-4" \
23
  --per_device_train_batch_size="2" \
24
  --per_device_eval_batch_size="2" \
25
  --save_total_limit="5"\
 
16
  --adam_beta2="0.98" \
17
  --logging_steps="250" \
18
  --eval_steps="500" \
19
+ --num_train_epochs="5" \
20
  --preprocessing_num_workers="96" \
21
+ --save_steps="500" \
22
+ --learning_rate="5e-5" \
23
  --per_device_train_batch_size="2" \
24
  --per_device_eval_batch_size="2" \
25
  --save_total_limit="5"\
run_mlm_flax.py CHANGED
@@ -59,7 +59,7 @@ from flax.training import checkpoints
59
  from flax.jax_utils import unreplicate
60
  from flax.training.checkpoints import save_checkpoint, restore_checkpoint
61
  from importlib.util import find_spec
62
-
63
 
64
  MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
65
  MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
 
59
  from flax.jax_utils import unreplicate
60
  from flax.training.checkpoints import save_checkpoint, restore_checkpoint
61
  from importlib.util import find_spec
62
+ from flax.serialization import to_bytes, from_bytes
63
 
64
  MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
65
  MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
wandb/debug-internal.log CHANGED
@@ -1 +1 @@
1
- run-20210714_210351-1msvb4w4/logs/debug-internal.log
 
1
+ run-20210714_212350-6qjir2uu/logs/debug-internal.log
wandb/debug.log CHANGED
@@ -1 +1 @@
1
- run-20210714_210351-1msvb4w4/logs/debug.log
 
1
+ run-20210714_212350-6qjir2uu/logs/debug.log
wandb/latest-run CHANGED
@@ -1 +1 @@
1
- run-20210714_210351-1msvb4w4
 
1
+ run-20210714_212350-6qjir2uu
wandb/run-20210714_210351-1msvb4w4/files/config.yaml CHANGED
@@ -13,6 +13,9 @@ _wandb:
13
  1:
14
  - 3
15
  - 11
 
 
 
16
  4: 3.8.10
17
  5: 0.10.33
18
  6: 4.9.0.dev0
 
13
  1:
14
  - 3
15
  - 11
16
+ 2:
17
+ - 3
18
+ - 11
19
  4: 3.8.10
20
  5: 0.10.33
21
  6: 4.9.0.dev0
wandb/run-20210714_210351-1msvb4w4/files/output.log CHANGED
@@ -416,3 +416,13 @@ Step... (1000 | Loss: 9.379836082458496, Acc: 0.047341905534267426): 33%|██
416
  Training...: 0%|▏ | 2/1250 [00:02<02:18, 9.03it/s]
417
  [21:21:14] - INFO - huggingface_hub.repository - git version 2.25.1 | 2/1250 [00:02<02:18, 9.03it/s]
418
  git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
 
 
 
 
 
 
 
 
 
 
 
416
  Training...: 0%|▏ | 2/1250 [00:02<02:18, 9.03it/s]
417
  [21:21:14] - INFO - huggingface_hub.repository - git version 2.25.1 | 2/1250 [00:02<02:18, 9.03it/s]
418
  git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
419
+ [21:21:14] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
420
+ [21:22:01] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (27/27), 1.3 GB | 56 MB/s, done.
421
+ Training...: 0%|▏ | 2/1250 [00:51<9:00:35, 25.99s/it]
422
+ Step... (1000 | Loss: 9.379836082458496, Acc: 0.047341905534267426): 33%|█████████████████████████ | 1/3 [17:55<35:50, 1075.43s/it]
423
+ Traceback (most recent call last):
424
+ File "./run_mlm_flax.py", line 850, in <module>
425
+ save_model_checkpoint(model, training_args.output_dir, state, with_opt=model_args.save_optimizer,
426
+ File "./run_mlm_flax.py", line 461, in save_model_checkpoint
427
+ f.write(to_bytes(state.opt_state))
428
+ NameError: name 'to_bytes' is not defined
wandb/run-20210714_210351-1msvb4w4/logs/debug-internal.log CHANGED
@@ -642,3 +642,112 @@
642
  2021-07-14 21:21:24,199 DEBUG SenderThread:576692 [sender.py:send():179] send: stats
643
  2021-07-14 21:21:33,465 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: stop_status
644
  2021-07-14 21:21:33,466 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: stop_status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
642
  2021-07-14 21:21:24,199 DEBUG SenderThread:576692 [sender.py:send():179] send: stats
643
  2021-07-14 21:21:33,465 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: stop_status
644
  2021-07-14 21:21:33,466 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: stop_status
645
+ 2021-07-14 21:21:48,598 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: stop_status
646
+ 2021-07-14 21:21:48,598 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: stop_status
647
+ 2021-07-14 21:21:54,276 DEBUG SenderThread:576692 [sender.py:send():179] send: stats
648
+ 2021-07-14 21:22:03,999 INFO Thread-8 :576692 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/output.log
649
+ 2021-07-14 21:22:04,937 DEBUG SenderThread:576692 [sender.py:send():179] send: telemetry
650
+ 2021-07-14 21:22:04,938 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
651
+ 2021-07-14 21:22:04,938 DEBUG SenderThread:576692 [sender.py:send():179] send: exit
652
+ 2021-07-14 21:22:04,938 INFO SenderThread:576692 [sender.py:send_exit():287] handling exit code: 1
653
+ 2021-07-14 21:22:04,940 INFO SenderThread:576692 [sender.py:send_exit():295] send defer
654
+ 2021-07-14 21:22:04,940 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
655
+ 2021-07-14 21:22:04,941 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
656
+ 2021-07-14 21:22:04,941 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 0
657
+ 2021-07-14 21:22:04,941 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
658
+ 2021-07-14 21:22:04,941 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 0
659
+ 2021-07-14 21:22:04,941 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 1
660
+ 2021-07-14 21:22:04,942 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
661
+ 2021-07-14 21:22:04,942 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 1
662
+ 2021-07-14 21:22:04,999 INFO Thread-8 :576692 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/output.log
663
+ 2021-07-14 21:22:05,004 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
664
+ 2021-07-14 21:22:05,004 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 1
665
+ 2021-07-14 21:22:05,004 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 2
666
+ 2021-07-14 21:22:05,004 DEBUG SenderThread:576692 [sender.py:send():179] send: stats
667
+ 2021-07-14 21:22:05,005 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
668
+ 2021-07-14 21:22:05,005 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 2
669
+ 2021-07-14 21:22:05,005 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
670
+ 2021-07-14 21:22:05,005 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 2
671
+ 2021-07-14 21:22:05,005 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 3
672
+ 2021-07-14 21:22:05,006 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
673
+ 2021-07-14 21:22:05,006 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 3
674
+ 2021-07-14 21:22:05,006 DEBUG SenderThread:576692 [sender.py:send():179] send: summary
675
+ 2021-07-14 21:22:05,007 INFO SenderThread:576692 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
676
+ 2021-07-14 21:22:05,007 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
677
+ 2021-07-14 21:22:05,007 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 3
678
+ 2021-07-14 21:22:05,007 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 4
679
+ 2021-07-14 21:22:05,007 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
680
+ 2021-07-14 21:22:05,007 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 4
681
+ 2021-07-14 21:22:05,007 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
682
+ 2021-07-14 21:22:05,008 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 4
683
+ 2021-07-14 21:22:05,043 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
684
+ 2021-07-14 21:22:05,215 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 5
685
+ 2021-07-14 21:22:05,215 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
686
+ 2021-07-14 21:22:05,216 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
687
+ 2021-07-14 21:22:05,216 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 5
688
+ 2021-07-14 21:22:05,216 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
689
+ 2021-07-14 21:22:05,216 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 5
690
+ 2021-07-14 21:22:05,216 INFO SenderThread:576692 [dir_watcher.py:finish():282] shutting down directory watcher
691
+ 2021-07-14 21:22:05,317 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
692
+ 2021-07-14 21:22:06,000 INFO Thread-8 :576692 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/config.yaml
693
+ 2021-07-14 21:22:06,000 INFO SenderThread:576692 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/wandb-summary.json
694
+ 2021-07-14 21:22:06,001 INFO SenderThread:576692 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files
695
+ 2021-07-14 21:22:06,001 INFO SenderThread:576692 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/requirements.txt requirements.txt
696
+ 2021-07-14 21:22:06,001 INFO SenderThread:576692 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/output.log output.log
697
+ 2021-07-14 21:22:06,001 INFO SenderThread:576692 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/wandb-metadata.json wandb-metadata.json
698
+ 2021-07-14 21:22:06,001 INFO SenderThread:576692 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/config.yaml config.yaml
699
+ 2021-07-14 21:22:06,001 INFO SenderThread:576692 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/wandb-summary.json wandb-summary.json
700
+ 2021-07-14 21:22:06,002 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 6
701
+ 2021-07-14 21:22:06,003 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
702
+ 2021-07-14 21:22:06,003 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
703
+ 2021-07-14 21:22:06,003 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 6
704
+ 2021-07-14 21:22:06,006 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
705
+ 2021-07-14 21:22:06,006 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 6
706
+ 2021-07-14 21:22:06,007 INFO SenderThread:576692 [file_pusher.py:finish():177] shutting down file pusher
707
+ 2021-07-14 21:22:06,105 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
708
+ 2021-07-14 21:22:06,105 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
709
+ 2021-07-14 21:22:06,207 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
710
+ 2021-07-14 21:22:06,207 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
711
+ 2021-07-14 21:22:06,309 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
712
+ 2021-07-14 21:22:06,309 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
713
+ 2021-07-14 21:22:06,411 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
714
+ 2021-07-14 21:22:06,411 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
715
+ 2021-07-14 21:22:06,455 INFO Thread-12 :576692 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/requirements.txt
716
+ 2021-07-14 21:22:06,463 INFO Thread-15 :576692 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/wandb-summary.json
717
+ 2021-07-14 21:22:06,495 INFO Thread-13 :576692 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/output.log
718
+ 2021-07-14 21:22:06,512 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
719
+ 2021-07-14 21:22:06,513 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
720
+ 2021-07-14 21:22:06,514 INFO Thread-14 :576692 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/files/config.yaml
721
+ 2021-07-14 21:22:06,615 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
722
+ 2021-07-14 21:22:06,615 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
723
+ 2021-07-14 21:22:06,715 INFO Thread-7 :576692 [sender.py:transition_state():308] send defer: 7
724
+ 2021-07-14 21:22:06,715 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
725
+ 2021-07-14 21:22:06,716 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 7
726
+ 2021-07-14 21:22:06,716 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
727
+ 2021-07-14 21:22:06,716 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 7
728
+ 2021-07-14 21:22:06,718 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
729
+ 2021-07-14 21:22:06,999 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 8
730
+ 2021-07-14 21:22:06,999 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
731
+ 2021-07-14 21:22:07,000 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
732
+ 2021-07-14 21:22:07,000 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 8
733
+ 2021-07-14 21:22:07,000 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
734
+ 2021-07-14 21:22:07,000 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 8
735
+ 2021-07-14 21:22:07,000 INFO SenderThread:576692 [sender.py:transition_state():308] send defer: 9
736
+ 2021-07-14 21:22:07,001 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: defer
737
+ 2021-07-14 21:22:07,001 DEBUG SenderThread:576692 [sender.py:send():179] send: final
738
+ 2021-07-14 21:22:07,001 INFO HandlerThread:576692 [handler.py:handle_request_defer():141] handle defer: 9
739
+ 2021-07-14 21:22:07,001 DEBUG SenderThread:576692 [sender.py:send():179] send: footer
740
+ 2021-07-14 21:22:07,002 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: defer
741
+ 2021-07-14 21:22:07,002 INFO SenderThread:576692 [sender.py:send_request_defer():304] handle sender defer: 9
742
+ 2021-07-14 21:22:07,101 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: poll_exit
743
+ 2021-07-14 21:22:07,101 DEBUG SenderThread:576692 [sender.py:send_request():193] send_request: poll_exit
744
+ 2021-07-14 21:22:07,101 INFO SenderThread:576692 [file_pusher.py:join():182] waiting for file pusher
745
+ 2021-07-14 21:22:07,103 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: get_summary
746
+ 2021-07-14 21:22:07,104 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: sampled_history
747
+ 2021-07-14 21:22:07,105 DEBUG HandlerThread:576692 [handler.py:handle_request():124] handle_request: shutdown
748
+ 2021-07-14 21:22:07,105 INFO HandlerThread:576692 [handler.py:finish():638] shutting down handler
749
+ 2021-07-14 21:22:08,002 INFO WriterThread:576692 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210714_210351-1msvb4w4/run-1msvb4w4.wandb
750
+ 2021-07-14 21:22:08,102 INFO SenderThread:576692 [sender.py:finish():945] shutting down sender
751
+ 2021-07-14 21:22:08,102 INFO SenderThread:576692 [file_pusher.py:finish():177] shutting down file pusher
752
+ 2021-07-14 21:22:08,102 INFO SenderThread:576692 [file_pusher.py:join():182] waiting for file pusher
753
+ 2021-07-14 21:22:08,105 INFO MainThread:576692 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210714_210351-1msvb4w4/logs/debug.log CHANGED
@@ -23,3 +23,99 @@ config: {}
23
  2021-07-14 21:03:53,599 INFO MainThread:575437 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-03-44_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 1250, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
  2021-07-14 21:03:53,601 INFO MainThread:575437 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True}
25
  2021-07-14 21:03:53,602 INFO MainThread:575437 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  2021-07-14 21:03:53,599 INFO MainThread:575437 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-03-44_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 1250, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
  2021-07-14 21:03:53,601 INFO MainThread:575437 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True}
25
  2021-07-14 21:03:53,602 INFO MainThread:575437 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
26
+ 2021-07-14 21:22:02,467 INFO MainThread:575437 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1
27
+ 2021-07-14 21:22:02,468 INFO MainThread:575437 [wandb_run.py:_restore():1565] restore
28
+ 2021-07-14 21:22:04,942 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
29
+ wandb_count: 1
30
+ }
31
+ pusher_stats {
32
+ uploaded_bytes: 1443
33
+ total_bytes: 1443
34
+ }
35
+
36
+ 2021-07-14 21:22:05,216 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
37
+ wandb_count: 1
38
+ }
39
+ pusher_stats {
40
+ uploaded_bytes: 1443
41
+ total_bytes: 1443
42
+ }
43
+
44
+ 2021-07-14 21:22:06,003 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
45
+ wandb_count: 5
46
+ }
47
+ pusher_stats {
48
+ uploaded_bytes: 1443
49
+ total_bytes: 12951
50
+ }
51
+
52
+ 2021-07-14 21:22:06,106 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
53
+ wandb_count: 5
54
+ }
55
+ pusher_stats {
56
+ uploaded_bytes: 1443
57
+ total_bytes: 12951
58
+ }
59
+
60
+ 2021-07-14 21:22:06,208 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
61
+ wandb_count: 5
62
+ }
63
+ pusher_stats {
64
+ uploaded_bytes: 12951
65
+ total_bytes: 12951
66
+ }
67
+
68
+ 2021-07-14 21:22:06,310 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
69
+ wandb_count: 5
70
+ }
71
+ pusher_stats {
72
+ uploaded_bytes: 12951
73
+ total_bytes: 12951
74
+ }
75
+
76
+ 2021-07-14 21:22:06,412 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
77
+ wandb_count: 5
78
+ }
79
+ pusher_stats {
80
+ uploaded_bytes: 12951
81
+ total_bytes: 12951
82
+ }
83
+
84
+ 2021-07-14 21:22:06,514 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
85
+ wandb_count: 5
86
+ }
87
+ pusher_stats {
88
+ uploaded_bytes: 12951
89
+ total_bytes: 12951
90
+ }
91
+
92
+ 2021-07-14 21:22:06,616 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
93
+ wandb_count: 5
94
+ }
95
+ pusher_stats {
96
+ uploaded_bytes: 12951
97
+ total_bytes: 12951
98
+ }
99
+
100
+ 2021-07-14 21:22:07,000 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
101
+ wandb_count: 5
102
+ }
103
+ pusher_stats {
104
+ uploaded_bytes: 12951
105
+ total_bytes: 12951
106
+ }
107
+
108
+ 2021-07-14 21:22:07,102 INFO MainThread:575437 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
109
+ exit_result {
110
+ }
111
+ file_counts {
112
+ wandb_count: 5
113
+ }
114
+ pusher_stats {
115
+ uploaded_bytes: 12951
116
+ total_bytes: 12951
117
+ }
118
+
119
+ 2021-07-14 21:22:08,392 INFO MainThread:575437 [wandb_run.py:_show_summary():1870] rendering summary
120
+ 2021-07-14 21:22:08,392 INFO MainThread:575437 [wandb_run.py:_show_history():1908] rendering history
121
+ 2021-07-14 21:22:08,393 INFO MainThread:575437 [wandb_run.py:_show_files():1937] logging synced files
wandb/run-20210714_210351-1msvb4w4/run-1msvb4w4.wandb CHANGED
Binary files a/wandb/run-20210714_210351-1msvb4w4/run-1msvb4w4.wandb and b/wandb/run-20210714_210351-1msvb4w4/run-1msvb4w4.wandb differ
 
wandb/run-20210714_212350-6qjir2uu/files/config.yaml ADDED
@@ -0,0 +1,304 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ cli_version: 0.10.33
7
+ framework: huggingface
8
+ huggingface_version: 4.9.0.dev0
9
+ is_jupyter_run: false
10
+ is_kaggle_kernel: false
11
+ python_version: 3.8.10
12
+ t:
13
+ 1:
14
+ - 3
15
+ - 11
16
+ 4: 3.8.10
17
+ 5: 0.10.33
18
+ 6: 4.9.0.dev0
19
+ 8:
20
+ - 5
21
+ adafactor:
22
+ desc: null
23
+ value: false
24
+ adam_beta1:
25
+ desc: null
26
+ value: 0.9
27
+ adam_beta2:
28
+ desc: null
29
+ value: 0.98
30
+ adam_epsilon:
31
+ desc: null
32
+ value: 1.0e-08
33
+ cache_dir:
34
+ desc: null
35
+ value: null
36
+ config_name:
37
+ desc: null
38
+ value: ./
39
+ dataloader_drop_last:
40
+ desc: null
41
+ value: false
42
+ dataloader_num_workers:
43
+ desc: null
44
+ value: 0
45
+ dataloader_pin_memory:
46
+ desc: null
47
+ value: true
48
+ dataset_config_name:
49
+ desc: null
50
+ value: null
51
+ dataset_name:
52
+ desc: null
53
+ value: null
54
+ ddp_find_unused_parameters:
55
+ desc: null
56
+ value: null
57
+ debug:
58
+ desc: null
59
+ value: []
60
+ deepspeed:
61
+ desc: null
62
+ value: null
63
+ disable_tqdm:
64
+ desc: null
65
+ value: false
66
+ do_eval:
67
+ desc: null
68
+ value: false
69
+ do_predict:
70
+ desc: null
71
+ value: false
72
+ do_train:
73
+ desc: null
74
+ value: false
75
+ dtype:
76
+ desc: null
77
+ value: float32
78
+ eval_accumulation_steps:
79
+ desc: null
80
+ value: null
81
+ eval_steps:
82
+ desc: null
83
+ value: 500
84
+ evaluation_strategy:
85
+ desc: null
86
+ value: IntervalStrategy.NO
87
+ fp16:
88
+ desc: null
89
+ value: false
90
+ fp16_backend:
91
+ desc: null
92
+ value: auto
93
+ fp16_full_eval:
94
+ desc: null
95
+ value: false
96
+ fp16_opt_level:
97
+ desc: null
98
+ value: O1
99
+ gradient_accumulation_steps:
100
+ desc: null
101
+ value: 4
102
+ greater_is_better:
103
+ desc: null
104
+ value: null
105
+ group_by_length:
106
+ desc: null
107
+ value: false
108
+ ignore_data_skip:
109
+ desc: null
110
+ value: false
111
+ label_names:
112
+ desc: null
113
+ value: null
114
+ label_smoothing_factor:
115
+ desc: null
116
+ value: 0.0
117
+ learning_rate:
118
+ desc: null
119
+ value: 5.0e-05
120
+ length_column_name:
121
+ desc: null
122
+ value: length
123
+ line_by_line:
124
+ desc: null
125
+ value: false
126
+ load_best_model_at_end:
127
+ desc: null
128
+ value: false
129
+ local_rank:
130
+ desc: null
131
+ value: -1
132
+ log_level:
133
+ desc: null
134
+ value: -1
135
+ log_level_replica:
136
+ desc: null
137
+ value: -1
138
+ log_on_each_node:
139
+ desc: null
140
+ value: true
141
+ logging_dir:
142
+ desc: null
143
+ value: ./runs/Jul14_21-23-42_t1v-n-f5c06ea1-w-0
144
+ logging_first_step:
145
+ desc: null
146
+ value: false
147
+ logging_steps:
148
+ desc: null
149
+ value: 250
150
+ logging_strategy:
151
+ desc: null
152
+ value: IntervalStrategy.STEPS
153
+ lr_scheduler_type:
154
+ desc: null
155
+ value: SchedulerType.LINEAR
156
+ max_eval_samples:
157
+ desc: null
158
+ value: 500
159
+ max_grad_norm:
160
+ desc: null
161
+ value: 1.0
162
+ max_seq_length:
163
+ desc: null
164
+ value: 4096
165
+ max_steps:
166
+ desc: null
167
+ value: -1
168
+ metric_for_best_model:
169
+ desc: null
170
+ value: null
171
+ mlm_probability:
172
+ desc: null
173
+ value: 0.15
174
+ model_name_or_path:
175
+ desc: null
176
+ value: null
177
+ model_type:
178
+ desc: null
179
+ value: big_bird
180
+ mp_parameters:
181
+ desc: null
182
+ value: ''
183
+ no_cuda:
184
+ desc: null
185
+ value: false
186
+ num_train_epochs:
187
+ desc: null
188
+ value: 5.0
189
+ output_dir:
190
+ desc: null
191
+ value: ./
192
+ overwrite_cache:
193
+ desc: null
194
+ value: false
195
+ overwrite_output_dir:
196
+ desc: null
197
+ value: true
198
+ pad_to_max_length:
199
+ desc: null
200
+ value: false
201
+ past_index:
202
+ desc: null
203
+ value: -1
204
+ per_device_eval_batch_size:
205
+ desc: null
206
+ value: 2
207
+ per_device_train_batch_size:
208
+ desc: null
209
+ value: 2
210
+ per_gpu_eval_batch_size:
211
+ desc: null
212
+ value: null
213
+ per_gpu_train_batch_size:
214
+ desc: null
215
+ value: null
216
+ prediction_loss_only:
217
+ desc: null
218
+ value: false
219
+ preprocessing_num_workers:
220
+ desc: null
221
+ value: 96
222
+ push_to_hub:
223
+ desc: null
224
+ value: true
225
+ push_to_hub_model_id:
226
+ desc: null
227
+ value: ''
228
+ push_to_hub_organization:
229
+ desc: null
230
+ value: null
231
+ push_to_hub_token:
232
+ desc: null
233
+ value: null
234
+ remove_unused_columns:
235
+ desc: null
236
+ value: true
237
+ report_to:
238
+ desc: null
239
+ value:
240
+ - tensorboard
241
+ - wandb
242
+ resume_from_checkpoint:
243
+ desc: null
244
+ value: null
245
+ run_name:
246
+ desc: null
247
+ value: ./
248
+ save_on_each_node:
249
+ desc: null
250
+ value: false
251
+ save_optimizer:
252
+ desc: null
253
+ value: true
254
+ save_steps:
255
+ desc: null
256
+ value: 500
257
+ save_strategy:
258
+ desc: null
259
+ value: IntervalStrategy.STEPS
260
+ save_total_limit:
261
+ desc: null
262
+ value: 5
263
+ seed:
264
+ desc: null
265
+ value: 42
266
+ sharded_ddp:
267
+ desc: null
268
+ value: []
269
+ skip_memory_metrics:
270
+ desc: null
271
+ value: true
272
+ tokenizer_name:
273
+ desc: null
274
+ value: ./
275
+ tpu_metrics_debug:
276
+ desc: null
277
+ value: false
278
+ tpu_num_cores:
279
+ desc: null
280
+ value: null
281
+ train_ref_file:
282
+ desc: null
283
+ value: null
284
+ use_fast_tokenizer:
285
+ desc: null
286
+ value: true
287
+ use_legacy_prediction_loop:
288
+ desc: null
289
+ value: false
290
+ validation_ref_file:
291
+ desc: null
292
+ value: null
293
+ validation_split_percentage:
294
+ desc: null
295
+ value: 5
296
+ warmup_ratio:
297
+ desc: null
298
+ value: 0.0
299
+ warmup_steps:
300
+ desc: null
301
+ value: 5000
302
+ weight_decay:
303
+ desc: null
304
+ value: 0.0095
wandb/run-20210714_212350-6qjir2uu/files/output.log ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [21:24:05] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
2
+ /home/dat/pino/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py:3132: UserWarning: Explicitly requested dtype <class 'jax._src.numpy.lax_numpy.int64'> requested in zeros is not available, and will be truncated to dtype int32. To enable more dtypes, set the jax_enable_x64 configuration option or the JAX_ENABLE_X64 shell environment variable. See https://github.com/google/jax#current-gotchas for more.
3
+ lax._check_user_dtype_supported(dtype, "zeros")
4
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
5
+ warnings.warn(
6
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
7
+ warnings.warn(
8
+ Epoch ... (1/5): 0%| | 0/5 [00:00<?, ?it/s][21:24:06] - INFO - __main__ - Skipping to epoch 0 step 0
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
52
+
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
65
+
66
+
67
+
68
+
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+
80
+
81
+
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+ Training...: 20%|█████████████████████████▌ | 253/1250 [04:26<1:03:55, 3.85s/it]
90
+
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+
120
+
121
+
122
+
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+ Training...: 40%|███████████████████████████████████████████████████▏ | 500/1250 [07:26<09:01, 1.39it/s]
169
+ Evaluating ...: 0%| | 0/31 [00:00<?, ?it/s]
170
+
171
+
172
+ [21:32:05] - INFO - huggingface_hub.repository - git version 2.25.1██████████████████████████████████████████████████████████████████████████████████| 31/31 [00:21<00:00, 9.98it/s]
173
+ git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
wandb/run-20210714_212350-6qjir2uu/files/requirements.txt ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.13.0
2
+ aiohttp==3.7.4.post0
3
+ astunparse==1.6.3
4
+ async-timeout==3.0.1
5
+ attrs==21.2.0
6
+ cachetools==4.2.2
7
+ certifi==2021.5.30
8
+ chardet==4.0.0
9
+ charset-normalizer==2.0.1
10
+ chex==0.0.8
11
+ click==8.0.1
12
+ configparser==5.0.2
13
+ cycler==0.10.0
14
+ datasets==1.9.1.dev0
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.0.12
19
+ flatbuffers==1.12
20
+ flax==0.3.4
21
+ fsspec==2021.7.0
22
+ gast==0.4.0
23
+ gitdb==4.0.7
24
+ gitpython==3.1.18
25
+ google-auth-oauthlib==0.4.4
26
+ google-auth==1.32.1
27
+ google-pasta==0.2.0
28
+ grpcio==1.34.1
29
+ h5py==3.1.0
30
+ huggingface-hub==0.0.12
31
+ idna==3.2
32
+ install==1.3.4
33
+ jax==0.2.17
34
+ jaxlib==0.1.68
35
+ joblib==1.0.1
36
+ keras-nightly==2.5.0.dev2021032900
37
+ keras-preprocessing==1.1.2
38
+ kiwisolver==1.3.1
39
+ libtpu-nightly==0.1.dev20210615
40
+ markdown==3.3.4
41
+ matplotlib==3.4.2
42
+ msgpack==1.0.2
43
+ multidict==5.1.0
44
+ multiprocess==0.70.12.2
45
+ numpy==1.19.5
46
+ oauthlib==3.1.1
47
+ opt-einsum==3.3.0
48
+ optax==0.0.9
49
+ packaging==21.0
50
+ pandas==1.3.0
51
+ pathtools==0.1.2
52
+ pillow==8.3.1
53
+ pip==20.0.2
54
+ pkg-resources==0.0.0
55
+ promise==2.3
56
+ protobuf==3.17.3
57
+ psutil==5.8.0
58
+ pyarrow==4.0.1
59
+ pyasn1-modules==0.2.8
60
+ pyasn1==0.4.8
61
+ pyparsing==2.4.7
62
+ python-dateutil==2.8.1
63
+ pytz==2021.1
64
+ pyyaml==5.4.1
65
+ regex==2021.7.6
66
+ requests-oauthlib==1.3.0
67
+ requests==2.26.0
68
+ rsa==4.7.2
69
+ sacremoses==0.0.45
70
+ scipy==1.7.0
71
+ sentry-sdk==1.3.0
72
+ setuptools==44.0.0
73
+ shortuuid==1.0.1
74
+ six==1.15.0
75
+ smmap==4.0.0
76
+ subprocess32==3.5.4
77
+ tensorboard-data-server==0.6.1
78
+ tensorboard-plugin-wit==1.8.0
79
+ tensorboard==2.5.0
80
+ tensorflow-estimator==2.5.0
81
+ tensorflow==2.5.0
82
+ termcolor==1.1.0
83
+ tokenizers==0.10.3
84
+ toolz==0.11.1
85
+ tqdm==4.61.2
86
+ transformers==4.9.0.dev0
87
+ typing-extensions==3.7.4.3
88
+ urllib3==1.26.6
89
+ wandb==0.10.33
90
+ werkzeug==2.0.1
91
+ wheel==0.36.2
92
+ wrapt==1.12.1
93
+ xxhash==2.0.2
94
+ yarl==1.6.3
wandb/run-20210714_212350-6qjir2uu/files/wandb-metadata.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-07-14T21:23:52.677692",
5
+ "startedAt": "2021-07-14T21:23:50.674053",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--push_to_hub",
11
+ "--output_dir=./",
12
+ "--model_type=big_bird",
13
+ "--config_name=./",
14
+ "--tokenizer_name=./",
15
+ "--max_seq_length=4096",
16
+ "--weight_decay=0.0095",
17
+ "--warmup_steps=5000",
18
+ "--overwrite_output_dir",
19
+ "--adam_beta1=0.9",
20
+ "--adam_beta2=0.98",
21
+ "--logging_steps=250",
22
+ "--eval_steps=500",
23
+ "--num_train_epochs=5",
24
+ "--preprocessing_num_workers=96",
25
+ "--save_steps=500",
26
+ "--learning_rate=5e-5",
27
+ "--per_device_train_batch_size=2",
28
+ "--per_device_eval_batch_size=2",
29
+ "--save_total_limit=5",
30
+ "--max_eval_samples=500",
31
+ "--overwrite_cache",
32
+ "False",
33
+ "--gradient_accumulation_steps=4"
34
+ ],
35
+ "state": "running",
36
+ "program": "./run_mlm_flax.py",
37
+ "codePath": "run_mlm_flax.py",
38
+ "git": {
39
+ "remote": "https://huggingface.co/flax-community/pino-roberta-base",
40
+ "commit": "f291f93b5d8c073340d75428272392e4c343a5ec"
41
+ },
42
+ "email": null,
43
+ "root": "/home/dat/pino-roberta-base",
44
+ "host": "t1v-n-f5c06ea1-w-0",
45
+ "username": "dat",
46
+ "executable": "/home/dat/pino/bin/python"
47
+ }
wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"training_step": 500, "learning_rate": 1.2499986041802913e-06, "train_loss": 10.137009620666504, "_runtime": 494, "_timestamp": 1626298324, "_step": 2, "eval_step": 500, "eval_accuracy": 0.043713752180337906, "eval_loss": 10.108721733093262}
wandb/run-20210714_212350-6qjir2uu/logs/debug-internal.log ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-14 21:23:51,351 INFO MainThread:580021 [internal.py:wandb_internal():88] W&B internal server running at pid: 580021, started at: 2021-07-14 21:23:51.351484
2
+ 2021-07-14 21:23:51,353 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: check_version
3
+ 2021-07-14 21:23:51,354 INFO WriterThread:580021 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/run-6qjir2uu.wandb
4
+ 2021-07-14 21:23:51,355 DEBUG SenderThread:580021 [sender.py:send():179] send: header
5
+ 2021-07-14 21:23:51,355 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: check_version
6
+ 2021-07-14 21:23:51,392 DEBUG SenderThread:580021 [sender.py:send():179] send: run
7
+ 2021-07-14 21:23:51,563 INFO SenderThread:580021 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files
8
+ 2021-07-14 21:23:51,563 INFO SenderThread:580021 [sender.py:_start_run_threads():716] run started: 6qjir2uu with start time 1626297830
9
+ 2021-07-14 21:23:51,563 DEBUG SenderThread:580021 [sender.py:send():179] send: summary
10
+ 2021-07-14 21:23:51,563 INFO SenderThread:580021 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
11
+ 2021-07-14 21:23:51,563 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: run_start
12
+ 2021-07-14 21:23:52,572 INFO Thread-8 :580021 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json
13
+ 2021-07-14 21:23:52,677 DEBUG HandlerThread:580021 [meta.py:__init__():39] meta init
14
+ 2021-07-14 21:23:52,677 DEBUG HandlerThread:580021 [meta.py:__init__():53] meta init done
15
+ 2021-07-14 21:23:52,677 DEBUG HandlerThread:580021 [meta.py:probe():210] probe
16
+ 2021-07-14 21:23:52,678 DEBUG HandlerThread:580021 [meta.py:_setup_git():200] setup git
17
+ 2021-07-14 21:23:52,708 DEBUG HandlerThread:580021 [meta.py:_setup_git():207] setup git done
18
+ 2021-07-14 21:23:52,708 DEBUG HandlerThread:580021 [meta.py:_save_pip():57] save pip
19
+ 2021-07-14 21:23:52,709 DEBUG HandlerThread:580021 [meta.py:_save_pip():71] save pip done
20
+ 2021-07-14 21:23:52,709 DEBUG HandlerThread:580021 [meta.py:probe():252] probe done
21
+ 2021-07-14 21:23:52,712 DEBUG SenderThread:580021 [sender.py:send():179] send: files
22
+ 2021-07-14 21:23:52,712 INFO SenderThread:580021 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
23
+ 2021-07-14 21:23:52,718 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
24
+ 2021-07-14 21:23:52,718 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
25
+ 2021-07-14 21:23:52,863 DEBUG SenderThread:580021 [sender.py:send():179] send: config
26
+ 2021-07-14 21:23:52,864 DEBUG SenderThread:580021 [sender.py:send():179] send: config
27
+ 2021-07-14 21:23:52,864 DEBUG SenderThread:580021 [sender.py:send():179] send: config
28
+ 2021-07-14 21:23:53,163 INFO Thread-11 :580021 [upload_job.py:push():137] Uploaded file /tmp/tmpzpobzd3rwandb/1kqe8dfg-wandb-metadata.json
29
+ 2021-07-14 21:23:53,571 INFO Thread-8 :580021 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-metadata.json
30
+ 2021-07-14 21:23:53,571 INFO Thread-8 :580021 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
31
+ 2021-07-14 21:23:53,571 INFO Thread-8 :580021 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/requirements.txt
32
+ 2021-07-14 21:24:07,576 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
33
+ 2021-07-14 21:24:07,865 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
34
+ 2021-07-14 21:24:07,865 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
35
+ 2021-07-14 21:24:13,578 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
36
+ 2021-07-14 21:24:20,750 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
37
+ 2021-07-14 21:24:22,581 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/config.yaml
38
+ 2021-07-14 21:24:23,011 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
39
+ 2021-07-14 21:24:23,011 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
40
+ 2021-07-14 21:24:38,163 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
41
+ 2021-07-14 21:24:38,163 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
42
+ 2021-07-14 21:24:50,833 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
43
+ 2021-07-14 21:24:53,294 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
44
+ 2021-07-14 21:24:53,294 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
45
+ 2021-07-14 21:25:08,436 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
46
+ 2021-07-14 21:25:08,437 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
47
+ 2021-07-14 21:25:20,911 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
48
+ 2021-07-14 21:25:23,576 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
49
+ 2021-07-14 21:25:23,576 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
50
+ 2021-07-14 21:25:37,608 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
51
+ 2021-07-14 21:25:38,709 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
52
+ 2021-07-14 21:25:38,709 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
53
+ 2021-07-14 21:25:39,609 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
54
+ 2021-07-14 21:25:41,610 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
55
+ 2021-07-14 21:25:43,610 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
56
+ 2021-07-14 21:25:45,611 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
57
+ 2021-07-14 21:25:47,612 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
58
+ 2021-07-14 21:25:49,613 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
59
+ 2021-07-14 21:25:50,992 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
60
+ 2021-07-14 21:25:51,614 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
61
+ 2021-07-14 21:25:53,614 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
62
+ 2021-07-14 21:25:53,875 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
63
+ 2021-07-14 21:25:53,875 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
64
+ 2021-07-14 21:25:55,615 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
65
+ 2021-07-14 21:25:57,616 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
66
+ 2021-07-14 21:25:59,617 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
67
+ 2021-07-14 21:26:01,617 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
68
+ 2021-07-14 21:26:03,618 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
69
+ 2021-07-14 21:26:05,619 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
70
+ 2021-07-14 21:26:07,620 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
71
+ 2021-07-14 21:26:09,007 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
72
+ 2021-07-14 21:26:09,007 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
73
+ 2021-07-14 21:26:09,621 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
74
+ 2021-07-14 21:26:11,622 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
75
+ 2021-07-14 21:26:13,623 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
76
+ 2021-07-14 21:26:15,624 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
77
+ 2021-07-14 21:26:17,625 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
78
+ 2021-07-14 21:26:19,625 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
79
+ 2021-07-14 21:26:21,071 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
80
+ 2021-07-14 21:26:21,626 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
81
+ 2021-07-14 21:26:23,627 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
82
+ 2021-07-14 21:26:24,137 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
83
+ 2021-07-14 21:26:24,137 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
84
+ 2021-07-14 21:26:25,628 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
85
+ 2021-07-14 21:26:27,629 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
86
+ 2021-07-14 21:26:29,629 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
87
+ 2021-07-14 21:26:31,630 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
88
+ 2021-07-14 21:26:33,631 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
89
+ 2021-07-14 21:26:35,632 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
90
+ 2021-07-14 21:26:37,633 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
91
+ 2021-07-14 21:26:39,266 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
92
+ 2021-07-14 21:26:39,266 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
93
+ 2021-07-14 21:26:39,633 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
94
+ 2021-07-14 21:26:41,634 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
95
+ 2021-07-14 21:26:43,635 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
96
+ 2021-07-14 21:26:45,636 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
97
+ 2021-07-14 21:26:47,637 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
98
+ 2021-07-14 21:26:49,637 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
99
+ 2021-07-14 21:26:51,148 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
100
+ 2021-07-14 21:26:51,638 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
101
+ 2021-07-14 21:26:53,639 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
102
+ 2021-07-14 21:26:54,396 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
103
+ 2021-07-14 21:26:54,396 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
104
+ 2021-07-14 21:26:55,640 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
105
+ 2021-07-14 21:26:57,641 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
106
+ 2021-07-14 21:26:59,641 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
107
+ 2021-07-14 21:27:01,642 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
108
+ 2021-07-14 21:27:03,643 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
109
+ 2021-07-14 21:27:05,644 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
110
+ 2021-07-14 21:27:07,644 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
111
+ 2021-07-14 21:27:09,528 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
112
+ 2021-07-14 21:27:09,528 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
113
+ 2021-07-14 21:27:09,645 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
114
+ 2021-07-14 21:27:11,646 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
115
+ 2021-07-14 21:27:13,647 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
116
+ 2021-07-14 21:27:15,648 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
117
+ 2021-07-14 21:27:17,649 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
118
+ 2021-07-14 21:27:19,649 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
119
+ 2021-07-14 21:27:21,220 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
120
+ 2021-07-14 21:27:21,650 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
121
+ 2021-07-14 21:27:23,651 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
122
+ 2021-07-14 21:27:24,659 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
123
+ 2021-07-14 21:27:24,659 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
124
+ 2021-07-14 21:27:25,652 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
125
+ 2021-07-14 21:27:27,653 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
126
+ 2021-07-14 21:27:29,653 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
127
+ 2021-07-14 21:27:31,654 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
128
+ 2021-07-14 21:27:33,655 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
129
+ 2021-07-14 21:27:35,656 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
130
+ 2021-07-14 21:27:37,656 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
131
+ 2021-07-14 21:27:39,657 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
132
+ 2021-07-14 21:27:39,790 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
133
+ 2021-07-14 21:27:39,790 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
134
+ 2021-07-14 21:27:41,658 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
135
+ 2021-07-14 21:27:43,659 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
136
+ 2021-07-14 21:27:46,660 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
137
+ 2021-07-14 21:27:48,661 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
138
+ 2021-07-14 21:27:50,662 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
139
+ 2021-07-14 21:27:51,295 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
140
+ 2021-07-14 21:27:52,662 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
141
+ 2021-07-14 21:27:54,663 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
142
+ 2021-07-14 21:27:54,922 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
143
+ 2021-07-14 21:27:54,923 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
144
+ 2021-07-14 21:27:56,664 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
145
+ 2021-07-14 21:27:58,665 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
146
+ 2021-07-14 21:28:00,666 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
147
+ 2021-07-14 21:28:02,667 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
148
+ 2021-07-14 21:28:04,668 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
149
+ 2021-07-14 21:28:06,668 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
150
+ 2021-07-14 21:28:08,669 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
151
+ 2021-07-14 21:28:10,056 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
152
+ 2021-07-14 21:28:10,057 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
153
+ 2021-07-14 21:28:10,670 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
154
+ 2021-07-14 21:28:12,671 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
155
+ 2021-07-14 21:28:14,671 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
156
+ 2021-07-14 21:28:21,367 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
157
+ 2021-07-14 21:28:25,189 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
158
+ 2021-07-14 21:28:25,189 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
159
+ 2021-07-14 21:28:37,228 DEBUG SenderThread:580021 [sender.py:send():179] send: history
160
+ 2021-07-14 21:28:37,229 DEBUG SenderThread:580021 [sender.py:send():179] send: summary
161
+ 2021-07-14 21:28:37,231 INFO SenderThread:580021 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
162
+ 2021-07-14 21:28:37,680 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json
163
+ 2021-07-14 21:28:38,681 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
164
+ 2021-07-14 21:28:39,681 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
165
+ 2021-07-14 21:28:40,320 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
166
+ 2021-07-14 21:28:40,320 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
167
+ 2021-07-14 21:28:40,681 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
168
+ 2021-07-14 21:28:42,682 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
169
+ 2021-07-14 21:28:44,683 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
170
+ 2021-07-14 21:28:46,684 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
171
+ 2021-07-14 21:28:48,685 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
172
+ 2021-07-14 21:28:50,686 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
173
+ 2021-07-14 21:28:51,442 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
174
+ 2021-07-14 21:28:52,687 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
175
+ 2021-07-14 21:28:54,687 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
176
+ 2021-07-14 21:28:55,463 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
177
+ 2021-07-14 21:28:55,463 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
178
+ 2021-07-14 21:28:56,688 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
179
+ 2021-07-14 21:28:58,689 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
180
+ 2021-07-14 21:29:00,690 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
181
+ 2021-07-14 21:29:02,691 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
182
+ 2021-07-14 21:29:04,691 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
183
+ 2021-07-14 21:29:06,692 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
184
+ 2021-07-14 21:29:08,693 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
185
+ 2021-07-14 21:29:10,610 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
186
+ 2021-07-14 21:29:10,611 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
187
+ 2021-07-14 21:29:10,694 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
188
+ 2021-07-14 21:29:12,695 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
189
+ 2021-07-14 21:29:14,695 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
190
+ 2021-07-14 21:29:16,696 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
191
+ 2021-07-14 21:29:18,697 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
192
+ 2021-07-14 21:29:20,698 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
193
+ 2021-07-14 21:29:21,521 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
194
+ 2021-07-14 21:29:22,699 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
195
+ 2021-07-14 21:29:24,700 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
196
+ 2021-07-14 21:29:25,747 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
197
+ 2021-07-14 21:29:25,748 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
198
+ 2021-07-14 21:29:26,701 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
199
+ 2021-07-14 21:29:28,702 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
200
+ 2021-07-14 21:29:30,703 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
201
+ 2021-07-14 21:29:32,704 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
202
+ 2021-07-14 21:29:34,705 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
203
+ 2021-07-14 21:29:36,706 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
204
+ 2021-07-14 21:29:38,707 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
205
+ 2021-07-14 21:29:40,707 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
206
+ 2021-07-14 21:29:40,883 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
207
+ 2021-07-14 21:29:40,883 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
208
+ 2021-07-14 21:29:42,708 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
209
+ 2021-07-14 21:29:44,709 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
210
+ 2021-07-14 21:29:46,710 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
211
+ 2021-07-14 21:29:48,711 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
212
+ 2021-07-14 21:29:50,712 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
213
+ 2021-07-14 21:29:51,599 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
214
+ 2021-07-14 21:29:52,713 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
215
+ 2021-07-14 21:29:54,714 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
216
+ 2021-07-14 21:29:56,025 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
217
+ 2021-07-14 21:29:56,025 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
218
+ 2021-07-14 21:29:56,715 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
219
+ 2021-07-14 21:29:58,715 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
220
+ 2021-07-14 21:30:00,716 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
221
+ 2021-07-14 21:30:02,717 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
222
+ 2021-07-14 21:30:04,718 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
223
+ 2021-07-14 21:30:06,719 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
224
+ 2021-07-14 21:30:08,720 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
225
+ 2021-07-14 21:30:10,721 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
226
+ 2021-07-14 21:30:11,164 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
227
+ 2021-07-14 21:30:11,165 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
228
+ 2021-07-14 21:30:12,722 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
229
+ 2021-07-14 21:30:14,723 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
230
+ 2021-07-14 21:30:16,724 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
231
+ 2021-07-14 21:30:18,724 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
232
+ 2021-07-14 21:30:20,725 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
233
+ 2021-07-14 21:30:21,667 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
234
+ 2021-07-14 21:30:22,726 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
235
+ 2021-07-14 21:30:24,727 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
236
+ 2021-07-14 21:30:26,298 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
237
+ 2021-07-14 21:30:26,298 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
238
+ 2021-07-14 21:30:26,728 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
239
+ 2021-07-14 21:30:28,729 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
240
+ 2021-07-14 21:30:30,730 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
241
+ 2021-07-14 21:30:33,731 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
242
+ 2021-07-14 21:30:35,732 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
243
+ 2021-07-14 21:30:37,733 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
244
+ 2021-07-14 21:30:39,734 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
245
+ 2021-07-14 21:30:41,436 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
246
+ 2021-07-14 21:30:41,437 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
247
+ 2021-07-14 21:30:41,735 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
248
+ 2021-07-14 21:30:43,736 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
249
+ 2021-07-14 21:30:45,737 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
250
+ 2021-07-14 21:30:47,738 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
251
+ 2021-07-14 21:30:49,739 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
252
+ 2021-07-14 21:30:51,734 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
253
+ 2021-07-14 21:30:51,740 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
254
+ 2021-07-14 21:30:53,741 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
255
+ 2021-07-14 21:30:55,742 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
256
+ 2021-07-14 21:30:56,575 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
257
+ 2021-07-14 21:30:56,576 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
258
+ 2021-07-14 21:30:57,742 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
259
+ 2021-07-14 21:30:59,743 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
260
+ 2021-07-14 21:31:01,744 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
261
+ 2021-07-14 21:31:03,745 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
262
+ 2021-07-14 21:31:05,746 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
263
+ 2021-07-14 21:31:07,747 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
264
+ 2021-07-14 21:31:09,748 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
265
+ 2021-07-14 21:31:11,717 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
266
+ 2021-07-14 21:31:11,717 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
267
+ 2021-07-14 21:31:11,749 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
268
+ 2021-07-14 21:31:13,749 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
269
+ 2021-07-14 21:31:15,750 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
270
+ 2021-07-14 21:31:21,803 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
271
+ 2021-07-14 21:31:26,853 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
272
+ 2021-07-14 21:31:26,853 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
273
+ 2021-07-14 21:31:37,846 DEBUG SenderThread:580021 [sender.py:send():179] send: history
274
+ 2021-07-14 21:31:37,847 DEBUG SenderThread:580021 [sender.py:send():179] send: summary
275
+ 2021-07-14 21:31:37,847 INFO SenderThread:580021 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
276
+ 2021-07-14 21:31:38,759 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json
277
+ 2021-07-14 21:31:39,760 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
278
+ 2021-07-14 21:31:41,981 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
279
+ 2021-07-14 21:31:41,982 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
280
+ 2021-07-14 21:31:51,878 DEBUG SenderThread:580021 [sender.py:send():179] send: stats
281
+ 2021-07-14 21:31:57,123 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
282
+ 2021-07-14 21:31:57,123 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
283
+ 2021-07-14 21:31:59,768 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
284
+ 2021-07-14 21:32:01,769 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
285
+ 2021-07-14 21:32:04,447 DEBUG SenderThread:580021 [sender.py:send():179] send: history
286
+ 2021-07-14 21:32:04,448 DEBUG SenderThread:580021 [sender.py:send():179] send: summary
287
+ 2021-07-14 21:32:04,448 INFO SenderThread:580021 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
288
+ 2021-07-14 21:32:04,770 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/wandb-summary.json
289
+ 2021-07-14 21:32:07,771 INFO Thread-8 :580021 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/files/output.log
290
+ 2021-07-14 21:32:12,254 DEBUG HandlerThread:580021 [handler.py:handle_request():124] handle_request: stop_status
291
+ 2021-07-14 21:32:12,255 DEBUG SenderThread:580021 [sender.py:send_request():193] send_request: stop_status
wandb/run-20210714_212350-6qjir2uu/logs/debug.log ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-14 21:23:50,675 INFO MainThread:578764 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-14 21:23:50,675 INFO MainThread:578764 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-14 21:23:50,675 INFO MainThread:578764 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/logs/debug.log
4
+ 2021-07-14 21:23:50,675 INFO MainThread:578764 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210714_212350-6qjir2uu/logs/debug-internal.log
5
+ 2021-07-14 21:23:50,675 INFO MainThread:578764 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-14 21:23:50,676 INFO MainThread:578764 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-14 21:23:50,676 INFO MainThread:578764 [wandb_init.py:init():419] starting backend
9
+ 2021-07-14 21:23:50,676 INFO MainThread:578764 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-14 21:23:50,721 INFO MainThread:578764 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-14 21:23:50,766 INFO MainThread:578764 [backend.py:ensure_launched():139] started backend process with pid: 580021
12
+ 2021-07-14 21:23:50,768 INFO MainThread:578764 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-14 21:23:50,772 INFO MainThread:578764 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-14 21:23:50,772 INFO MainThread:578764 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-14 21:23:51,391 INFO MainThread:578764 [wandb_init.py:init():496] got version response
16
+ 2021-07-14 21:23:51,391 INFO MainThread:578764 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
17
+ 2021-07-14 21:23:51,563 INFO MainThread:578764 [wandb_init.py:init():529] starting run threads in backend
18
+ 2021-07-14 21:23:52,715 INFO MainThread:578764 [wandb_run.py:_console_start():1623] atexit reg
19
+ 2021-07-14 21:23:52,716 INFO MainThread:578764 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
20
+ 2021-07-14 21:23:52,716 INFO MainThread:578764 [wandb_run.py:_redirect():1502] Redirecting console.
21
+ 2021-07-14 21:23:52,718 INFO MainThread:578764 [wandb_run.py:_redirect():1558] Redirects installed.
22
+ 2021-07-14 21:23:52,718 INFO MainThread:578764 [wandb_init.py:init():554] run started, returning control to user process
23
+ 2021-07-14 21:23:52,724 INFO MainThread:578764 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 2, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 5000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul14_21-23-42_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 250, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
+ 2021-07-14 21:23:52,726 INFO MainThread:578764 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32', 'save_optimizer': True}
25
+ 2021-07-14 21:23:52,727 INFO MainThread:578764 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
wandb/run-20210714_212350-6qjir2uu/run-6qjir2uu.wandb ADDED
Binary file (52.2 kB). View file