Hugging Face's logo

flax-community
/
pino-bigbird-roberta-base

Fill-Mask Transformers PyTorch JAX TensorBoard
Dutch big_bird Inference Endpoints
Model card Files Metrics Community
2
dat commited on
Commit
dc01ff5
•
1 Parent(s): 0e50dbb

Saving weights and logs of step 240000

Browse files
Files changed (22) hide show
  1. checkpoint_240000 +3 -0
  2. create_pt_model.py +1 -1
  3. events.out.tfevents.1626397185.t1v-n-f5c06ea1-w-0.760449.3.v2 +2 -2
  4. events.out.tfevents.1626429561.t1v-n-f5c06ea1-w-0.782479.3.v2 +3 -0
  5. flax_model.msgpack +1 -1
  6. run.sh +1 -1
  7. wandb/debug-internal.log +1 -1
  8. wandb/debug.log +1 -1
  9. wandb/latest-run +1 -1
  10. wandb/run-20210716_005946-3bte92ft/files/output.log +120 -0
  11. wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +1 -1
  12. wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log +269 -0
  13. wandb/run-20210716_005946-3bte92ft/logs/debug.log +2 -0
  14. wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb +0 -0
  15. wandb/run-20210716_095921-13hxxunp/files/config.yaml +308 -0
  16. wandb/run-20210716_095921-13hxxunp/files/output.log +1779 -0
  17. wandb/run-20210716_095921-13hxxunp/files/requirements.txt +95 -0
  18. wandb/run-20210716_095921-13hxxunp/files/wandb-metadata.json +45 -0
  19. wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +1 -0
  20. wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log +0 -0
  21. wandb/run-20210716_095921-13hxxunp/logs/debug.log +26 -0
  22. wandb/run-20210716_095921-13hxxunp/run-13hxxunp.wandb +0 -0
checkpoint_240000 ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:829a6abc0c8b1c376c211be682dc46f6a7d7fbc1ddf4114262e4e1cf169cde42
3
+ size 1530270447
create_pt_model.py CHANGED
@@ -2,4 +2,4 @@ from transformers import BigBirdForMaskedLM
2
  import torch
3
 
4
  pt_model = BigBirdForMaskedLM.from_pretrained(".", from_flax=True)
5
- pt_model.save_pretrained(".")
2
  import torch
3
 
4
  pt_model = BigBirdForMaskedLM.from_pretrained(".", from_flax=True)
5
+ pt_model.save_pretrained(".",save_config=False)
events.out.tfevents.1626397185.t1v-n-f5c06ea1-w-0.760449.3.v2 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:55333c246f74b2c84620675b6048997ae79474af6bfdd07693a18bed98c164cc
3
- size 11281744
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1c6fce6e15405613b7ad5d5c8a11bab2c7731406ba245e3113f9ef8e63e75f6
3
+ size 11424694
events.out.tfevents.1626429561.t1v-n-f5c06ea1-w-0.782479.3.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45a436bec44dfea2809d645a06db2ef771283532bf125612badb26de71cec05c
3
+ size 2256466
flax_model.msgpack CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e8f5ca7b60c5e4b03edfdde3f172ec2572f02f5753f13fe7a658c57c2fc9bae5
3
  size 510090043
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ae1d45c74c0892af03af2f6156aa1d6c21cff644f407de47ad5b18a10b3c17c
3
  size 510090043
run.sh CHANGED
@@ -16,7 +16,7 @@ python ./run_mlm_flax_no_accum.py \
16
  --adam_beta2="0.98" \
17
  --logging_steps="50" \
18
  --eval_steps="6000" \
19
- --num_train_epochs="3"\
20
  --preprocessing_num_workers="96" \
21
  --save_steps="15000" \
22
  --learning_rate="3e-5" \
16
  --adam_beta2="0.98" \
17
  --logging_steps="50" \
18
  --eval_steps="6000" \
19
+ --num_train_epochs="5"\
20
  --preprocessing_num_workers="96" \
21
  --save_steps="15000" \
22
  --learning_rate="3e-5" \
wandb/debug-internal.log CHANGED
@@ -1 +1 @@
1
- run-20210716_005946-3bte92ft/logs/debug-internal.log
1
+ run-20210716_095921-13hxxunp/logs/debug-internal.log
wandb/debug.log CHANGED
@@ -1 +1 @@
1
- run-20210716_005946-3bte92ft/logs/debug.log
1
+ run-20210716_095921-13hxxunp/logs/debug.log
wandb/latest-run CHANGED
@@ -1 +1 @@
1
- run-20210716_005946-3bte92ft
1
+ run-20210716_095921-13hxxunp
wandb/run-20210716_005946-3bte92ft/files/output.log CHANGED
@@ -8608,3 +8608,123 @@ tcmalloc: large alloc 2715181056 bytes == 0x41e66a000 @ 0x7f779c64f680 0x7f779c
8608
  [09:50:33] - INFO - absl - Saved checkpoint at checkpoint_225000
8609
  [09:50:34] - INFO - huggingface_hub.repository - git version 2.25.1
8610
  git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8608
  [09:50:33] - INFO - absl - Saved checkpoint at checkpoint_225000
8609
  [09:50:34] - INFO - huggingface_hub.repository - git version 2.25.1
8610
  git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
8611
+ [09:50:35] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
8612
+ [09:51:44] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 2.1 GB | 46 MB/s, done.
8613
+
8614
+
8615
+
8616
+
8617
+ Training...: 75049it [8:46:15, 2.69it/s]
8618
+
8619
+
8620
+
8621
+
8622
+
8623
+ Training...: 75099it [8:46:35, 2.70it/s]
8624
+
8625
+
8626
+
8627
+
8628
+
8629
+ Training...: 75149it [8:46:55, 2.72it/s]
8630
+
8631
+
8632
+
8633
+
8634
+
8635
+ Training...: 75199it [8:47:15, 2.66it/s]
8636
+
8637
+
8638
+
8639
+
8640
+
8641
+ Training...: 75249it [8:47:35, 2.68it/s]
8642
+
8643
+
8644
+
8645
+
8646
+
8647
+ Training...: 75299it [8:47:55, 2.69it/s]
8648
+
8649
+
8650
+
8651
+
8652
+
8653
+ Training...: 75349it [8:48:15, 2.68it/s]
8654
+
8655
+
8656
+
8657
+
8658
+
8659
+ Training...: 75399it [8:48:35, 2.72it/s]
8660
+
8661
+
8662
+
8663
+
8664
+
8665
+ Training...: 75449it [8:48:55, 2.69it/s]
8666
+
8667
+
8668
+
8669
+
8670
+
8671
+ Training...: 75499it [8:49:15, 2.70it/s]
8672
+
8673
+
8674
+
8675
+
8676
+
8677
+ Training...: 75549it [8:49:36, 2.68it/s]
8678
+
8679
+
8680
+
8681
+
8682
+
8683
+ Training...: 75599it [8:49:56, 2.68it/s]
8684
+
8685
+
8686
+
8687
+
8688
+
8689
+ Training...: 75649it [8:50:16, 2.72it/s]
8690
+
8691
+
8692
+
8693
+
8694
+
8695
+ Training...: 75699it [8:50:36, 2.66it/s]
8696
+
8697
+
8698
+
8699
+
8700
+
8701
+ Training...: 75749it [8:50:56, 2.72it/s]
8702
+
8703
+
8704
+
8705
+
8706
+
8707
+ Training...: 75799it [8:51:16, 2.70it/s]
8708
+
8709
+
8710
+
8711
+
8712
+
8713
+ Training...: 75849it [8:51:36, 2.66it/s]
8714
+
8715
+
8716
+
8717
+
8718
+
8719
+ Training...: 75899it [8:51:56, 2.75it/s]
8720
+
8721
+
8722
+
8723
+
8724
+
8725
+ Training...: 75949it [8:52:16, 2.67it/s]
8726
+
8727
+
8728
+
8729
+
8730
+
wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"training_step": 225000, "learning_rate": 2.3847169359214604e-05, "train_loss": 2.061807155609131, "_runtime": 31840, "_timestamp": 1626429026, "_step": 1511, "eval_step": 222000, "eval_accuracy": 0.6153644919395447, "eval_loss": 1.9865816831588745}
1
+ {"training_step": 225950, "learning_rate": 2.3819982743589208e-05, "train_loss": 1.9876642227172852, "_runtime": 32300, "_timestamp": 1626429486, "_step": 1530, "eval_step": 222000, "eval_accuracy": 0.6153644919395447, "eval_loss": 1.9865816831588745}
wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log CHANGED
@@ -19048,3 +19048,272 @@
19048
  2021-07-16 09:50:37,587 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19049
  2021-07-16 09:50:37,588 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19050
  2021-07-16 09:50:38,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19048
  2021-07-16 09:50:37,587 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19049
  2021-07-16 09:50:37,588 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19050
  2021-07-16 09:50:38,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19051
+ 2021-07-16 09:50:52,724 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19052
+ 2021-07-16 09:50:52,724 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19053
+ 2021-07-16 09:51:04,008 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19054
+ 2021-07-16 09:51:07,872 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19055
+ 2021-07-16 09:51:07,872 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19056
+ 2021-07-16 09:51:23,004 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19057
+ 2021-07-16 09:51:23,004 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19058
+ 2021-07-16 09:51:34,084 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19059
+ 2021-07-16 09:51:38,251 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19060
+ 2021-07-16 09:51:38,252 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19061
+ 2021-07-16 09:51:46,091 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19062
+ 2021-07-16 09:51:48,092 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19063
+ 2021-07-16 09:51:50,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19064
+ 2021-07-16 09:51:52,094 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19065
+ 2021-07-16 09:51:53,384 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19066
+ 2021-07-16 09:51:53,384 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19067
+ 2021-07-16 09:51:54,094 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19068
+ 2021-07-16 09:52:04,156 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19069
+ 2021-07-16 09:52:05,407 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19070
+ 2021-07-16 09:52:05,407 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19071
+ 2021-07-16 09:52:05,408 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19072
+ 2021-07-16 09:52:06,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19073
+ 2021-07-16 09:52:06,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19074
+ 2021-07-16 09:52:08,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19075
+ 2021-07-16 09:52:08,515 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19076
+ 2021-07-16 09:52:08,515 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19077
+ 2021-07-16 09:52:10,100 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19078
+ 2021-07-16 09:52:13,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19079
+ 2021-07-16 09:52:15,102 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19080
+ 2021-07-16 09:52:23,654 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19081
+ 2021-07-16 09:52:23,655 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19082
+ 2021-07-16 09:52:25,445 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19083
+ 2021-07-16 09:52:25,445 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19084
+ 2021-07-16 09:52:25,446 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19085
+ 2021-07-16 09:52:26,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19086
+ 2021-07-16 09:52:26,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19087
+ 2021-07-16 09:52:27,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19088
+ 2021-07-16 09:52:29,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19089
+ 2021-07-16 09:52:31,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19090
+ 2021-07-16 09:52:33,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19091
+ 2021-07-16 09:52:34,233 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19092
+ 2021-07-16 09:52:35,110 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19093
+ 2021-07-16 09:52:38,789 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19094
+ 2021-07-16 09:52:38,789 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19095
+ 2021-07-16 09:52:45,483 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19096
+ 2021-07-16 09:52:45,484 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19097
+ 2021-07-16 09:52:45,484 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19098
+ 2021-07-16 09:52:46,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19099
+ 2021-07-16 09:52:46,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19100
+ 2021-07-16 09:52:47,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19101
+ 2021-07-16 09:52:49,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19102
+ 2021-07-16 09:52:51,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19103
+ 2021-07-16 09:52:53,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19104
+ 2021-07-16 09:52:53,952 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19105
+ 2021-07-16 09:52:53,952 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19106
+ 2021-07-16 09:52:55,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19107
+ 2021-07-16 09:53:04,307 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19108
+ 2021-07-16 09:53:05,595 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19109
+ 2021-07-16 09:53:05,596 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19110
+ 2021-07-16 09:53:05,596 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19111
+ 2021-07-16 09:53:06,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19112
+ 2021-07-16 09:53:07,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19113
+ 2021-07-16 09:53:09,081 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19114
+ 2021-07-16 09:53:09,082 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19115
+ 2021-07-16 09:53:09,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19116
+ 2021-07-16 09:53:11,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19117
+ 2021-07-16 09:53:13,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19118
+ 2021-07-16 09:53:15,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19119
+ 2021-07-16 09:53:24,213 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19120
+ 2021-07-16 09:53:24,214 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19121
+ 2021-07-16 09:53:25,635 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19122
+ 2021-07-16 09:53:25,635 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19123
+ 2021-07-16 09:53:25,636 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19124
+ 2021-07-16 09:53:26,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19125
+ 2021-07-16 09:53:27,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19126
+ 2021-07-16 09:53:29,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19127
+ 2021-07-16 09:53:31,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19128
+ 2021-07-16 09:53:33,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19129
+ 2021-07-16 09:53:34,383 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19130
+ 2021-07-16 09:53:35,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19131
+ 2021-07-16 09:53:39,343 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19132
+ 2021-07-16 09:53:39,343 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19133
+ 2021-07-16 09:53:45,684 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19134
+ 2021-07-16 09:53:45,685 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19135
+ 2021-07-16 09:53:45,685 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19136
+ 2021-07-16 09:53:46,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19137
+ 2021-07-16 09:53:47,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19138
+ 2021-07-16 09:53:49,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19139
+ 2021-07-16 09:53:51,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19140
+ 2021-07-16 09:53:53,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19141
+ 2021-07-16 09:53:54,471 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19142
+ 2021-07-16 09:53:54,471 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19143
+ 2021-07-16 09:53:55,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19144
+ 2021-07-16 09:54:04,461 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19145
+ 2021-07-16 09:54:05,722 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19146
+ 2021-07-16 09:54:05,722 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19147
+ 2021-07-16 09:54:05,723 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19148
+ 2021-07-16 09:54:06,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19149
+ 2021-07-16 09:54:07,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19150
+ 2021-07-16 09:54:09,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19151
+ 2021-07-16 09:54:09,602 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19152
+ 2021-07-16 09:54:09,603 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19153
+ 2021-07-16 09:54:11,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19154
+ 2021-07-16 09:54:13,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19155
+ 2021-07-16 09:54:15,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19156
+ 2021-07-16 09:54:24,736 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19157
+ 2021-07-16 09:54:24,736 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19158
+ 2021-07-16 09:54:25,763 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19159
+ 2021-07-16 09:54:25,763 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19160
+ 2021-07-16 09:54:25,764 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19161
+ 2021-07-16 09:54:26,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19162
+ 2021-07-16 09:54:27,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19163
+ 2021-07-16 09:54:29,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19164
+ 2021-07-16 09:54:31,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19165
+ 2021-07-16 09:54:33,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19166
+ 2021-07-16 09:54:34,537 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19167
+ 2021-07-16 09:54:35,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19168
+ 2021-07-16 09:54:39,870 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19169
+ 2021-07-16 09:54:39,870 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19170
+ 2021-07-16 09:54:45,806 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19171
+ 2021-07-16 09:54:45,806 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19172
+ 2021-07-16 09:54:45,807 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19173
+ 2021-07-16 09:54:46,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19174
+ 2021-07-16 09:54:47,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19175
+ 2021-07-16 09:54:49,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19176
+ 2021-07-16 09:54:51,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19177
+ 2021-07-16 09:54:53,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19178
+ 2021-07-16 09:54:55,001 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19179
+ 2021-07-16 09:54:55,001 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19180
+ 2021-07-16 09:54:55,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19181
+ 2021-07-16 09:55:04,608 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19182
+ 2021-07-16 09:55:05,846 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19183
+ 2021-07-16 09:55:05,846 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19184
+ 2021-07-16 09:55:05,847 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19185
+ 2021-07-16 09:55:06,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19186
+ 2021-07-16 09:55:07,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19187
+ 2021-07-16 09:55:09,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19188
+ 2021-07-16 09:55:10,134 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19189
+ 2021-07-16 09:55:10,134 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19190
+ 2021-07-16 09:55:11,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19191
+ 2021-07-16 09:55:13,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19192
+ 2021-07-16 09:55:15,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19193
+ 2021-07-16 09:55:25,266 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19194
+ 2021-07-16 09:55:25,267 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19195
+ 2021-07-16 09:55:25,884 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19196
+ 2021-07-16 09:55:25,884 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19197
+ 2021-07-16 09:55:25,885 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19198
+ 2021-07-16 09:55:26,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19199
+ 2021-07-16 09:55:27,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19200
+ 2021-07-16 09:55:29,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19201
+ 2021-07-16 09:55:31,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19202
+ 2021-07-16 09:55:33,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19203
+ 2021-07-16 09:55:34,684 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19204
+ 2021-07-16 09:55:35,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19205
+ 2021-07-16 09:55:40,396 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19206
+ 2021-07-16 09:55:40,396 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19207
+ 2021-07-16 09:55:45,925 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19208
+ 2021-07-16 09:55:45,925 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19209
+ 2021-07-16 09:55:45,926 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19210
+ 2021-07-16 09:55:46,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19211
+ 2021-07-16 09:55:47,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19212
+ 2021-07-16 09:55:49,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19213
+ 2021-07-16 09:55:51,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19214
+ 2021-07-16 09:55:53,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19215
+ 2021-07-16 09:55:55,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19216
+ 2021-07-16 09:55:55,529 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19217
+ 2021-07-16 09:55:55,530 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19218
+ 2021-07-16 09:56:04,762 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19219
+ 2021-07-16 09:56:05,966 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19220
+ 2021-07-16 09:56:05,966 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19221
+ 2021-07-16 09:56:05,967 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19222
+ 2021-07-16 09:56:06,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19223
+ 2021-07-16 09:56:07,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19224
+ 2021-07-16 09:56:09,198 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19225
+ 2021-07-16 09:56:10,664 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19226
+ 2021-07-16 09:56:10,664 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19227
+ 2021-07-16 09:56:11,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19228
+ 2021-07-16 09:56:13,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19229
+ 2021-07-16 09:56:15,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19230
+ 2021-07-16 09:56:25,796 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19231
+ 2021-07-16 09:56:25,796 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19232
+ 2021-07-16 09:56:26,066 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19233
+ 2021-07-16 09:56:26,066 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19234
+ 2021-07-16 09:56:26,067 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19235
+ 2021-07-16 09:56:26,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19236
+ 2021-07-16 09:56:27,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19237
+ 2021-07-16 09:56:29,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19238
+ 2021-07-16 09:56:31,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19239
+ 2021-07-16 09:56:33,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19240
+ 2021-07-16 09:56:34,837 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19241
+ 2021-07-16 09:56:35,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19242
+ 2021-07-16 09:56:40,928 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19243
+ 2021-07-16 09:56:40,928 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19244
+ 2021-07-16 09:56:46,118 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19245
+ 2021-07-16 09:56:46,118 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19246
+ 2021-07-16 09:56:46,119 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19247
+ 2021-07-16 09:56:46,213 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19248
+ 2021-07-16 09:56:47,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19249
+ 2021-07-16 09:56:49,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19250
+ 2021-07-16 09:56:51,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19251
+ 2021-07-16 09:56:53,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19252
+ 2021-07-16 09:56:55,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19253
+ 2021-07-16 09:56:56,061 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19254
+ 2021-07-16 09:56:56,061 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19255
+ 2021-07-16 09:57:04,910 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19256
+ 2021-07-16 09:57:06,160 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19257
+ 2021-07-16 09:57:06,160 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19258
+ 2021-07-16 09:57:06,160 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19259
+ 2021-07-16 09:57:06,221 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19260
+ 2021-07-16 09:57:07,222 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19261
+ 2021-07-16 09:57:09,223 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19262
+ 2021-07-16 09:57:11,191 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19263
+ 2021-07-16 09:57:11,191 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19264
+ 2021-07-16 09:57:11,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19265
+ 2021-07-16 09:57:13,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19266
+ 2021-07-16 09:57:15,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19267
+ 2021-07-16 09:57:26,198 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19268
+ 2021-07-16 09:57:26,198 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19269
+ 2021-07-16 09:57:26,199 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19270
+ 2021-07-16 09:57:26,230 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19271
+ 2021-07-16 09:57:26,613 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19272
+ 2021-07-16 09:57:26,613 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19273
+ 2021-07-16 09:57:27,230 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19274
+ 2021-07-16 09:57:29,231 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19275
+ 2021-07-16 09:57:31,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19276
+ 2021-07-16 09:57:33,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19277
+ 2021-07-16 09:57:34,985 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19278
+ 2021-07-16 09:57:35,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19279
+ 2021-07-16 09:57:41,780 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19280
+ 2021-07-16 09:57:41,780 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19281
+ 2021-07-16 09:57:46,293 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19282
+ 2021-07-16 09:57:46,293 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19283
+ 2021-07-16 09:57:46,294 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19284
+ 2021-07-16 09:57:47,239 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19285
+ 2021-07-16 09:57:47,239 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19286
+ 2021-07-16 09:57:49,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19287
+ 2021-07-16 09:57:51,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19288
+ 2021-07-16 09:57:53,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19289
+ 2021-07-16 09:57:55,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19290
+ 2021-07-16 09:57:56,915 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19291
+ 2021-07-16 09:57:56,915 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19292
+ 2021-07-16 09:58:05,063 DEBUG SenderThread:761704 [sender.py:send():179] send: stats
19293
+ 2021-07-16 09:58:06,338 DEBUG SenderThread:761704 [sender.py:send():179] send: history
19294
+ 2021-07-16 09:58:06,339 DEBUG SenderThread:761704 [sender.py:send():179] send: summary
19295
+ 2021-07-16 09:58:06,339 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
19296
+ 2021-07-16 09:58:07,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19297
+ 2021-07-16 09:58:07,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json
19298
+ 2021-07-16 09:58:09,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19299
+ 2021-07-16 09:58:11,248 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19300
+ 2021-07-16 09:58:12,051 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status
19301
+ 2021-07-16 09:58:12,051 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status
19302
+ 2021-07-16 09:58:13,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19303
+ 2021-07-16 09:58:15,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log
19304
+ 2021-07-16 09:58:23,978 WARNING MainThread:761704 [internal.py:wandb_internal():147] Internal process interrupt: 1
19305
+ 2021-07-16 09:58:24,417 WARNING MainThread:761704 [internal.py:wandb_internal():147] Internal process interrupt: 2
19306
+ 2021-07-16 09:58:24,417 ERROR MainThread:761704 [internal.py:wandb_internal():150] Internal process interrupted.
19307
+ 2021-07-16 09:58:25,070 INFO WriterThread:761704 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb
19308
+ 2021-07-16 09:58:25,188 INFO SenderThread:761704 [sender.py:finish():945] shutting down sender
19309
+ 2021-07-16 09:58:25,188 INFO SenderThread:761704 [dir_watcher.py:finish():282] shutting down directory watcher
19310
+ 2021-07-16 09:58:25,191 INFO HandlerThread:761704 [handler.py:finish():638] shutting down handler
19311
+ 2021-07-16 09:58:25,253 INFO SenderThread:761704 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files
19312
+ 2021-07-16 09:58:25,253 INFO SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/requirements.txt requirements.txt
19313
+ 2021-07-16 09:58:25,253 INFO SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log output.log
19314
+ 2021-07-16 09:58:25,254 INFO SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-metadata.json wandb-metadata.json
19315
+ 2021-07-16 09:58:25,254 INFO SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/config.yaml config.yaml
19316
+ 2021-07-16 09:58:25,254 INFO SenderThread:761704 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json wandb-summary.json
19317
+ 2021-07-16 09:58:25,254 INFO SenderThread:761704 [file_pusher.py:finish():177] shutting down file pusher
19318
+ 2021-07-16 09:58:25,254 INFO SenderThread:761704 [file_pusher.py:join():182] waiting for file pusher
19319
+ 2021-07-16 09:58:25,419 INFO MainThread:761704 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210716_005946-3bte92ft/logs/debug.log CHANGED
@@ -24,3 +24,5 @@ config: {}
24
  2021-07-16 00:59:48,518 INFO MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul16_00-59-37_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 15000, 'save_total_limit': 20, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 6000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
25
  2021-07-16 00:59:48,520 INFO MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
26
  2021-07-16 00:59:48,521 INFO MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 4000}
 
 
24
  2021-07-16 00:59:48,518 INFO MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul16_00-59-37_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 15000, 'save_total_limit': 20, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 6000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
25
  2021-07-16 00:59:48,520 INFO MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
26
  2021-07-16 00:59:48,521 INFO MainThread:760449 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 4000}
27
+ 2021-07-16 09:58:24,335 INFO MainThread:760449 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
28
+ 2021-07-16 09:58:24,336 INFO MainThread:760449 [wandb_run.py:_restore():1565] restore
wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb CHANGED
Binary files a/wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb and b/wandb/run-20210716_005946-3bte92ft/run-3bte92ft.wandb differ
wandb/run-20210716_095921-13hxxunp/files/config.yaml ADDED
@@ -0,0 +1,308 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ __cached__setup_devices:
4
+ desc: null
5
+ value: cpu
6
+ _n_gpu:
7
+ desc: null
8
+ value: 0
9
+ _wandb:
10
+ desc: null
11
+ value:
12
+ cli_version: 0.10.33
13
+ framework: huggingface
14
+ huggingface_version: 4.9.0.dev0
15
+ is_jupyter_run: false
16
+ is_kaggle_kernel: false
17
+ python_version: 3.8.10
18
+ t:
19
+ 1:
20
+ - 1
21
+ - 3
22
+ - 11
23
+ 4: 3.8.10
24
+ 5: 0.10.33
25
+ 6: 4.9.0.dev0
26
+ 8:
27
+ - 5
28
+ adafactor:
29
+ desc: null
30
+ value: false
31
+ adam_beta1:
32
+ desc: null
33
+ value: 0.9
34
+ adam_beta2:
35
+ desc: null
36
+ value: 0.98
37
+ adam_epsilon:
38
+ desc: null
39
+ value: 1.0e-08
40
+ cache_dir:
41
+ desc: null
42
+ value: null
43
+ config_name:
44
+ desc: null
45
+ value: ./
46
+ dataloader_drop_last:
47
+ desc: null
48
+ value: false
49
+ dataloader_num_workers:
50
+ desc: null
51
+ value: 0
52
+ dataloader_pin_memory:
53
+ desc: null
54
+ value: true
55
+ dataset_config_name:
56
+ desc: null
57
+ value: null
58
+ dataset_name:
59
+ desc: null
60
+ value: null
61
+ ddp_find_unused_parameters:
62
+ desc: null
63
+ value: null
64
+ debug:
65
+ desc: null
66
+ value: []
67
+ deepspeed:
68
+ desc: null
69
+ value: null
70
+ disable_tqdm:
71
+ desc: null
72
+ value: false
73
+ do_eval:
74
+ desc: null
75
+ value: false
76
+ do_predict:
77
+ desc: null
78
+ value: false
79
+ do_train:
80
+ desc: null
81
+ value: false
82
+ dtype:
83
+ desc: null
84
+ value: float32
85
+ eval_accumulation_steps:
86
+ desc: null
87
+ value: null
88
+ eval_steps:
89
+ desc: null
90
+ value: 6000
91
+ evaluation_strategy:
92
+ desc: null
93
+ value: IntervalStrategy.NO
94
+ fp16:
95
+ desc: null
96
+ value: false
97
+ fp16_backend:
98
+ desc: null
99
+ value: auto
100
+ fp16_full_eval:
101
+ desc: null
102
+ value: false
103
+ fp16_opt_level:
104
+ desc: null
105
+ value: O1
106
+ gradient_accumulation_steps:
107
+ desc: null
108
+ value: 1
109
+ greater_is_better:
110
+ desc: null
111
+ value: null
112
+ group_by_length:
113
+ desc: null
114
+ value: false
115
+ ignore_data_skip:
116
+ desc: null
117
+ value: false
118
+ label_names:
119
+ desc: null
120
+ value: null
121
+ label_smoothing_factor:
122
+ desc: null
123
+ value: 0.0
124
+ learning_rate:
125
+ desc: null
126
+ value: 3.0e-05
127
+ length_column_name:
128
+ desc: null
129
+ value: length
130
+ line_by_line:
131
+ desc: null
132
+ value: false
133
+ load_best_model_at_end:
134
+ desc: null
135
+ value: false
136
+ local_rank:
137
+ desc: null
138
+ value: -1
139
+ log_level:
140
+ desc: null
141
+ value: -1
142
+ log_level_replica:
143
+ desc: null
144
+ value: -1
145
+ log_on_each_node:
146
+ desc: null
147
+ value: true
148
+ logging_dir:
149
+ desc: null
150
+ value: ./runs/Jul16_09-59-13_t1v-n-f5c06ea1-w-0
151
+ logging_first_step:
152
+ desc: null
153
+ value: false
154
+ logging_steps:
155
+ desc: null
156
+ value: 50
157
+ logging_strategy:
158
+ desc: null
159
+ value: IntervalStrategy.STEPS
160
+ lr_scheduler_type:
161
+ desc: null
162
+ value: SchedulerType.LINEAR
163
+ max_eval_samples:
164
+ desc: null
165
+ value: 4000
166
+ max_grad_norm:
167
+ desc: null
168
+ value: 1.0
169
+ max_seq_length:
170
+ desc: null
171
+ value: 4096
172
+ max_steps:
173
+ desc: null
174
+ value: -1
175
+ metric_for_best_model:
176
+ desc: null
177
+ value: null
178
+ mlm_probability:
179
+ desc: null
180
+ value: 0.15
181
+ model_name_or_path:
182
+ desc: null
183
+ value: null
184
+ model_type:
185
+ desc: null
186
+ value: big_bird
187
+ mp_parameters:
188
+ desc: null
189
+ value: ''
190
+ no_cuda:
191
+ desc: null
192
+ value: false
193
+ num_train_epochs:
194
+ desc: null
195
+ value: 5.0
196
+ output_dir:
197
+ desc: null
198
+ value: ./
199
+ overwrite_cache:
200
+ desc: null
201
+ value: false
202
+ overwrite_output_dir:
203
+ desc: null
204
+ value: true
205
+ pad_to_max_length:
206
+ desc: null
207
+ value: false
208
+ past_index:
209
+ desc: null
210
+ value: -1
211
+ per_device_eval_batch_size:
212
+ desc: null
213
+ value: 1
214
+ per_device_train_batch_size:
215
+ desc: null
216
+ value: 1
217
+ per_gpu_eval_batch_size:
218
+ desc: null
219
+ value: null
220
+ per_gpu_train_batch_size:
221
+ desc: null
222
+ value: null
223
+ prediction_loss_only:
224
+ desc: null
225
+ value: false
226
+ preprocessing_num_workers:
227
+ desc: null
228
+ value: 96
229
+ push_to_hub:
230
+ desc: null
231
+ value: true
232
+ push_to_hub_model_id:
233
+ desc: null
234
+ value: ''
235
+ push_to_hub_organization:
236
+ desc: null
237
+ value: null
238
+ push_to_hub_token:
239
+ desc: null
240
+ value: null
241
+ remove_unused_columns:
242
+ desc: null
243
+ value: true
244
+ report_to:
245
+ desc: null
246
+ value:
247
+ - tensorboard
248
+ - wandb
249
+ resume_from_checkpoint:
250
+ desc: null
251
+ value: ./
252
+ run_name:
253
+ desc: null
254
+ value: ./
255
+ save_on_each_node:
256
+ desc: null
257
+ value: false
258
+ save_steps:
259
+ desc: null
260
+ value: 15000
261
+ save_strategy:
262
+ desc: null
263
+ value: IntervalStrategy.STEPS
264
+ save_total_limit:
265
+ desc: null
266
+ value: 20
267
+ seed:
268
+ desc: null
269
+ value: 42
270
+ sharded_ddp:
271
+ desc: null
272
+ value: []
273
+ skip_memory_metrics:
274
+ desc: null
275
+ value: true
276
+ tokenizer_name:
277
+ desc: null
278
+ value: ./
279
+ tpu_metrics_debug:
280
+ desc: null
281
+ value: false
282
+ tpu_num_cores:
283
+ desc: null
284
+ value: null
285
+ train_ref_file:
286
+ desc: null
287
+ value: null
288
+ use_fast_tokenizer:
289
+ desc: null
290
+ value: true
291
+ use_legacy_prediction_loop:
292
+ desc: null
293
+ value: false
294
+ validation_ref_file:
295
+ desc: null
296
+ value: null
297
+ validation_split_percentage:
298
+ desc: null
299
+ value: 5
300
+ warmup_ratio:
301
+ desc: null
302
+ value: 0.0
303
+ warmup_steps:
304
+ desc: null
305
+ value: 10000
306
+ weight_decay:
307
+ desc: null
308
+ value: 0.0095
wandb/run-20210716_095921-13hxxunp/files/output.log ADDED
@@ -0,0 +1,1779 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ [09:59:36] - INFO - absl - Restoring checkpoint from ./checkpoint_225000
3
+ tcmalloc: large alloc 1530273792 bytes == 0x9d58c000 @ 0x7fe91a70f680 0x7fe91a730824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fe91a5240b3 0x5f96de
4
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
5
+ warnings.warn(
6
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
7
+ warnings.warn(
8
+ Epoch ... (1/5): 0%| | 0/5 [00:00<?, ?it/s]
9
+ Training...: 0it [00:00, ?it/s]
10
+
11
+
12
+
13
+
14
+ Training...: 49it [04:22, 2.65it/s]
15
+
16
+
17
+
18
+
19
+
20
+ Training...: 100it [04:55, 4.01s/it]
21
+
22
+
23
+
24
+
25
+
26
+ Training...: 152it [05:15, 2.24s/it]
27
+
28
+
29
+
30
+
31
+
32
+ Training...: 202it [05:36, 2.25s/it]
33
+
34
+
35
+
36
+
37
+ Training...: 249it [05:42, 2.68it/s]
38
+
39
+
40
+
41
+
42
+ Training...: 299it [06:02, 2.67it/s]
43
+
44
+
45
+
46
+
47
+
48
+ Training...: 350it [06:35, 4.01s/it]
49
+
50
+
51
+
52
+
53
+
54
+ Training...: 400it [06:56, 4.01s/it]
55
+
56
+
57
+
58
+
59
+
60
+ Training...: 450it [07:16, 4.01s/it]
61
+
62
+
63
+
64
+
65
+
66
+ Training...: 502it [07:36, 2.24s/it]
67
+
68
+
69
+
70
+
71
+
72
+ Training...: 552it [07:56, 2.25s/it]
73
+
74
+
75
+
76
+
77
+
78
+ Training...: 604it [08:16, 1.39s/it]
79
+
80
+
81
+
82
+
83
+ Training...: 649it [08:23, 2.67it/s]
84
+
85
+
86
+
87
+
88
+ Training...: 699it [08:43, 2.69it/s]
89
+
90
+
91
+
92
+
93
+ Training...: 749it [09:03, 2.67it/s]
94
+
95
+
96
+
97
+
98
+ Training...: 799it [09:23, 2.68it/s]
99
+
100
+
101
+
102
+
103
+
104
+ Training...: 850it [09:56, 4.01s/it]
105
+
106
+
107
+
108
+
109
+
110
+ Training...: 900it [10:16, 4.01s/it]
111
+
112
+
113
+
114
+
115
+
116
+ Training...: 950it [10:36, 4.01s/it]
117
+
118
+
119
+
120
+
121
+
122
+ Training...: 1002it [10:56, 2.25s/it]
123
+
124
+
125
+
126
+
127
+
128
+ Training...: 1054it [11:16, 1.39s/it]
129
+
130
+
131
+
132
+
133
+
134
+ Training...: 1104it [11:36, 1.39s/it]
135
+
136
+
137
+
138
+
139
+ Training...: 1149it [11:43, 2.69it/s]
140
+
141
+
142
+
143
+
144
+ Training...: 1199it [12:03, 2.67it/s]
145
+
146
+
147
+
148
+
149
+ Training...: 1249it [12:23, 2.67it/s]
150
+
151
+
152
+
153
+
154
+ Training...: 1299it [12:43, 2.69it/s]
155
+
156
+
157
+
158
+
159
+ Training...: 1349it [13:03, 2.67it/s]
160
+
161
+
162
+
163
+
164
+
165
+ Training...: 1400it [13:36, 4.01s/it]
166
+
167
+
168
+
169
+
170
+
171
+ Training...: 1450it [13:56, 4.01s/it]
172
+
173
+
174
+
175
+
176
+
177
+ Training...: 1500it [14:16, 4.01s/it]
178
+
179
+
180
+
181
+
182
+
183
+ Training...: 1552it [14:37, 2.24s/it]
184
+
185
+
186
+
187
+
188
+
189
+ Training...: 1602it [14:57, 2.24s/it]
190
+
191
+
192
+
193
+
194
+
195
+ Training...: 1654it [15:17, 1.39s/it]
196
+
197
+
198
+
199
+
200
+ Training...: 1699it [15:23, 2.67it/s]
201
+
202
+
203
+
204
+
205
+ Training...: 1749it [15:43, 2.70it/s]
206
+
207
+
208
+
209
+
210
+ Training...: 1799it [16:03, 2.67it/s]
211
+
212
+
213
+
214
+
215
+
216
+ Training...: 1850it [16:37, 4.01s/it]
217
+
218
+
219
+
220
+
221
+
222
+ Training...: 1900it [16:57, 4.01s/it]
223
+
224
+
225
+
226
+
227
+
228
+ Training...: 1952it [17:17, 2.24s/it]
229
+
230
+
231
+
232
+
233
+
234
+ Training...: 2002it [17:37, 2.24s/it]
235
+
236
+
237
+
238
+
239
+
240
+ Training...: 2054it [17:57, 1.39s/it]
241
+
242
+
243
+
244
+
245
+
246
+ Training...: 2104it [18:17, 1.39s/it]
247
+
248
+
249
+
250
+
251
+ Training...: 2149it [18:24, 2.66it/s]
252
+
253
+
254
+
255
+
256
+ Training...: 2199it [18:44, 2.68it/s]
257
+
258
+
259
+
260
+
261
+
262
+ Training...: 2250it [19:17, 4.01s/it]
263
+
264
+
265
+
266
+
267
+
268
+ Training...: 2300it [19:37, 4.01s/it]
269
+
270
+
271
+
272
+
273
+
274
+ Training...: 2352it [19:57, 2.24s/it]
275
+
276
+
277
+
278
+
279
+
280
+ Training...: 2402it [20:17, 2.24s/it]
281
+
282
+
283
+
284
+
285
+
286
+ Training...: 2454it [20:37, 1.39s/it]
287
+
288
+
289
+
290
+
291
+
292
+ Training...: 2504it [20:57, 1.39s/it]
293
+
294
+
295
+
296
+
297
+ Training...: 2549it [21:04, 2.67it/s]
298
+
299
+
300
+
301
+
302
+ Training...: 2599it [21:24, 2.67it/s]
303
+
304
+
305
+
306
+
307
+ Training...: 2649it [21:44, 2.67it/s]
308
+
309
+
310
+
311
+
312
+
313
+ Training...: 2702it [22:17, 2.24s/it]
314
+
315
+
316
+
317
+
318
+
319
+ Training...: 2754it [22:38, 1.39s/it]
320
+
321
+
322
+
323
+
324
+
325
+ Training...: 2804it [22:58, 1.39s/it]
326
+
327
+
328
+
329
+
330
+
331
+ Training...: 2854it [23:18, 1.39s/it]
332
+
333
+
334
+
335
+
336
+
337
+ Training...: 2900it [23:37, 4.01s/it]
338
+
339
+
340
+
341
+
342
+
343
+ Training...: 2950it [23:57, 4.01s/it]
344
+
345
+
346
+
347
+
348
+
349
+ Training...: 2999it [24:17, 2.67it/s]
350
+ Step... (228000 | Loss: 2.156348943710327, Learning Rate: 2.6271016395185143e-05)
351
+
352
+
353
+
354
+
355
+
356
+
357
+
358
+
359
+
360
+
361
+
362
+
363
+
364
+
365
+
366
+
367
+
368
+
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+
379
+
380
+
381
+
382
+
383
+
384
+
385
+ Training...: 3049it [25:46, 2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
386
+
387
+
388
+
389
+
390
+ Training...: 3099it [26:06, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
391
+
392
+
393
+
394
+
395
+
396
+ Training...: 3166it [26:41, 5.39it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
397
+
398
+
399
+
400
+
401
+
402
+ Training...: 3208it [27:00, 1.59it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
403
+
404
+
405
+
406
+
407
+
408
+ Training...: 3258it [27:20, 1.59it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
409
+
410
+
411
+
412
+
413
+ Training...: 3299it [27:26, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
414
+
415
+
416
+
417
+
418
+
419
+ Training...: 3349it [27:47, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
420
+
421
+
422
+
423
+
424
+ Training...: 3399it [28:07, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
425
+
426
+
427
+
428
+
429
+ Training...: 3449it [28:27, 2.66it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
430
+
431
+
432
+
433
+
434
+
435
+ Training...: 3499it [28:47, 2.63it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
436
+
437
+
438
+
439
+
440
+
441
+ Training...: 3549it [29:07, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
442
+
443
+
444
+
445
+
446
+
447
+ Training...: 3599it [29:27, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
448
+
449
+
450
+
451
+
452
+
453
+ Training...: 3649it [29:47, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
454
+
455
+
456
+
457
+
458
+
459
+ Training...: 3699it [30:07, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
460
+
461
+
462
+
463
+
464
+ Training...: 3749it [30:27, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
465
+
466
+
467
+
468
+
469
+ Training...: 3799it [30:47, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
470
+
471
+
472
+
473
+
474
+ Training...: 3849it [31:07, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
475
+
476
+
477
+
478
+
479
+ Training...: 3899it [31:27, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
480
+
481
+
482
+
483
+
484
+ Training...: 3949it [31:47, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
485
+
486
+
487
+
488
+
489
+
490
+ Training...: 3999it [32:07, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
491
+
492
+
493
+
494
+
495
+
496
+ Training...: 4049it [32:27, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
497
+
498
+
499
+
500
+
501
+
502
+ Training...: 4099it [32:47, 2.63it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
503
+
504
+
505
+
506
+
507
+
508
+ Training...: 4149it [33:07, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
509
+
510
+
511
+
512
+
513
+
514
+ Training...: 4199it [33:27, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
515
+
516
+
517
+
518
+
519
+
520
+ Training...: 4249it [33:47, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
521
+
522
+
523
+
524
+
525
+
526
+ Training...: 4299it [34:07, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
527
+
528
+
529
+
530
+
531
+ Training...: 4349it [34:27, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
532
+
533
+
534
+
535
+
536
+
537
+ Training...: 4399it [34:47, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
538
+
539
+
540
+
541
+
542
+
543
+ Training...: 4449it [35:07, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
544
+
545
+
546
+
547
+
548
+
549
+ Training...: 4499it [35:27, 2.69it/s]███████████████████████████████��██████████| 500/500 [01:17<00:00, 7.90it/s]
550
+
551
+
552
+
553
+
554
+
555
+ Training...: 4549it [35:47, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
556
+
557
+
558
+
559
+
560
+
561
+ Training...: 4599it [36:07, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
562
+
563
+
564
+
565
+
566
+
567
+ Training...: 4649it [36:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
568
+
569
+
570
+
571
+
572
+ Training...: 4699it [36:48, 2.74it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
573
+
574
+
575
+
576
+
577
+ Training...: 4749it [37:08, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
578
+
579
+
580
+
581
+
582
+ Training...: 4799it [37:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
583
+
584
+
585
+
586
+
587
+ Training...: 4849it [37:48, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
588
+
589
+
590
+
591
+
592
+
593
+ Training...: 4899it [38:08, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
594
+
595
+
596
+
597
+
598
+
599
+ Training...: 4949it [38:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
600
+
601
+
602
+
603
+
604
+
605
+ Training...: 4999it [38:48, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
606
+
607
+
608
+
609
+
610
+
611
+ Training...: 5049it [39:08, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
612
+
613
+
614
+
615
+
616
+
617
+ Training...: 5099it [39:28, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
618
+
619
+
620
+
621
+
622
+
623
+ Training...: 5149it [39:48, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
624
+
625
+
626
+
627
+
628
+ Training...: 5199it [40:08, 2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
629
+
630
+
631
+
632
+
633
+ Training...: 5249it [40:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
634
+
635
+
636
+
637
+
638
+ Training...: 5299it [40:48, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
639
+
640
+
641
+
642
+
643
+ Training...: 5349it [41:08, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
644
+
645
+
646
+
647
+
648
+ Training...: 5399it [41:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
649
+
650
+
651
+
652
+
653
+
654
+ Training...: 5449it [41:48, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
655
+
656
+
657
+
658
+
659
+
660
+ Training...: 5499it [42:08, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
661
+
662
+
663
+
664
+
665
+
666
+ Training...: 5549it [42:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
667
+
668
+
669
+
670
+
671
+
672
+ Training...: 5599it [42:48, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
673
+
674
+
675
+
676
+
677
+
678
+ Training...: 5649it [43:08, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
679
+
680
+
681
+
682
+
683
+
684
+ Training...: 5699it [43:28, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
685
+
686
+
687
+
688
+
689
+ Training...: 5749it [43:48, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
690
+
691
+
692
+
693
+
694
+ Training...: 5799it [44:08, 2.66it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
695
+
696
+
697
+
698
+
699
+ Training...: 5849it [44:28, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
700
+
701
+
702
+
703
+
704
+ Training...: 5899it [44:48, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
705
+
706
+
707
+
708
+
709
+ Training...: 5949it [45:09, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
710
+
711
+
712
+
713
+
714
+ Training...: 5999it [45:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
715
+
716
+
717
+
718
+
719
+
720
+ Training...: 6049it [45:49, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
721
+
722
+
723
+
724
+
725
+ Training...: 6099it [46:09, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
726
+
727
+
728
+
729
+
730
+ Training...: 6149it [46:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
731
+
732
+
733
+
734
+
735
+ Training...: 6199it [46:49, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
736
+
737
+
738
+
739
+
740
+ Training...: 6249it [47:09, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
741
+
742
+
743
+
744
+
745
+
746
+ Training...: 6299it [47:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
747
+
748
+
749
+
750
+
751
+
752
+ Training...: 6349it [47:49, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
753
+
754
+
755
+
756
+
757
+
758
+ Training...: 6399it [48:09, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
759
+
760
+
761
+
762
+
763
+ Training...: 6449it [48:29, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
764
+
765
+
766
+
767
+
768
+ Training...: 6499it [48:49, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
769
+
770
+
771
+
772
+
773
+ Training...: 6549it [49:09, 2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
774
+
775
+
776
+
777
+
778
+ Training...: 6599it [49:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
779
+
780
+
781
+
782
+
783
+ Training...: 6649it [49:49, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
784
+
785
+
786
+
787
+
788
+ Training...: 6699it [50:09, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
789
+
790
+
791
+
792
+
793
+
794
+ Training...: 6749it [50:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
795
+
796
+
797
+
798
+
799
+
800
+ Training...: 6799it [50:49, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
801
+
802
+
803
+
804
+
805
+
806
+ Training...: 6849it [51:09, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
807
+
808
+
809
+
810
+
811
+
812
+ Training...: 6899it [51:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
813
+
814
+
815
+
816
+
817
+ Training...: 6949it [51:49, 2.73it/s]████████████████████████████████���█████████| 500/500 [01:17<00:00, 7.90it/s]
818
+
819
+
820
+
821
+
822
+ Training...: 6999it [52:09, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
823
+
824
+
825
+
826
+
827
+ Training...: 7049it [52:29, 2.71it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
828
+
829
+
830
+
831
+
832
+ Training...: 7099it [52:49, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
833
+
834
+
835
+
836
+
837
+ Training...: 7149it [53:09, 2.74it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
838
+
839
+
840
+
841
+
842
+ Training...: 7199it [53:29, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
843
+
844
+
845
+
846
+
847
+ Training...: 7249it [53:50, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
848
+
849
+
850
+
851
+
852
+ Training...: 7299it [54:10, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
853
+
854
+
855
+
856
+
857
+ Training...: 7349it [54:30, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
858
+
859
+
860
+
861
+
862
+ Training...: 7399it [54:50, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
863
+
864
+
865
+
866
+
867
+ Training...: 7449it [55:10, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
868
+
869
+
870
+
871
+
872
+ Training...: 7499it [55:30, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
873
+
874
+
875
+
876
+
877
+ Training...: 7549it [55:50, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
878
+
879
+
880
+
881
+
882
+ Training...: 7599it [56:10, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
883
+
884
+
885
+
886
+
887
+ Training...: 7649it [56:30, 2.68it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
888
+
889
+
890
+
891
+
892
+ Training...: 7699it [56:50, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
893
+
894
+
895
+
896
+
897
+ Training...: 7749it [57:10, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
898
+
899
+
900
+
901
+
902
+ Training...: 7799it [57:30, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
903
+
904
+
905
+
906
+
907
+ Training...: 7849it [57:50, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
908
+
909
+
910
+
911
+
912
+ Training...: 7899it [58:10, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
913
+
914
+
915
+
916
+
917
+ Training...: 7949it [58:30, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
918
+
919
+
920
+
921
+
922
+ Training...: 7999it [58:50, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
923
+
924
+
925
+
926
+
927
+ Training...: 8049it [59:10, 2.70it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
928
+
929
+
930
+
931
+
932
+ Training...: 8099it [59:30, 2.67it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
933
+
934
+
935
+
936
+
937
+ Training...: 8149it [59:50, 2.69it/s]██████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
938
+
939
+
940
+
941
+
942
+ Training...: 8199it [1:00:10, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
943
+
944
+
945
+
946
+
947
+ Training...: 8249it [1:00:30, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
948
+
949
+
950
+
951
+
952
+ Training...: 8299it [1:00:50, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
953
+
954
+
955
+
956
+
957
+ Training...: 8349it [1:01:10, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
958
+
959
+
960
+
961
+
962
+ Training...: 8399it [1:01:30, 2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
963
+
964
+
965
+
966
+
967
+ Training...: 8449it [1:01:50, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
968
+
969
+
970
+
971
+
972
+ Training...: 8499it [1:02:11, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
973
+
974
+
975
+
976
+
977
+ Training...: 8549it [1:02:31, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
978
+
979
+
980
+
981
+
982
+ Training...: 8599it [1:02:51, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
983
+
984
+
985
+
986
+
987
+ Training...: 8649it [1:03:11, 2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
988
+
989
+
990
+
991
+
992
+ Training...: 8699it [1:03:31, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
993
+
994
+
995
+
996
+
997
+ Training...: 8749it [1:03:51, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
998
+
999
+
1000
+
1001
+
1002
+ Training...: 8799it [1:04:11, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
1003
+
1004
+
1005
+
1006
+
1007
+ Training...: 8849it [1:04:31, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
1008
+
1009
+
1010
+
1011
+
1012
+ Training...: 8899it [1:04:51, 2.70it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
1013
+
1014
+
1015
+
1016
+
1017
+ Training...: 8949it [1:05:11, 2.67it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
1018
+
1019
+
1020
+
1021
+
1022
+ Training...: 8999it [1:05:31, 2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
1023
+ Training...: 8999it [1:05:44, 2.69it/s]████████████████████████████████████████| 500/500 [01:17<00:00, 7.90it/s]
1024
+
1025
+
1026
+
1027
+
1028
+
1029
+
1030
+
1031
+
1032
+
1033
+
1034
+
1035
+
1036
+
1037
+
1038
+
1039
+
1040
+
1041
+
1042
+
1043
+
1044
+
1045
+
1046
+
1047
+
1048
+
1049
+
1050
+
1051
+
1052
+
1053
+
1054
+
1055
+
1056
+
1057
+
1058
+
1059
+ Training...: 9050it [1:07:08, 4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1060
+
1061
+
1062
+
1063
+
1064
+
1065
+ Training...: 9102it [1:07:28, 2.25s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1066
+
1067
+
1068
+
1069
+
1070
+
1071
+ Training...: 9152it [1:07:48, 2.25s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1072
+
1073
+
1074
+
1075
+
1076
+
1077
+ Training...: 9204it [1:08:08, 1.39s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1078
+
1079
+
1080
+
1081
+
1082
+ Training...: 9249it [1:08:15, 2.69it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1083
+
1084
+
1085
+
1086
+
1087
+ Training...: 9299it [1:08:35, 2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1088
+
1089
+
1090
+
1091
+
1092
+ Training...: 9349it [1:08:55, 2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1093
+
1094
+
1095
+
1096
+
1097
+ Training...: 9399it [1:09:15, 2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1098
+
1099
+
1100
+
1101
+
1102
+ Training...: 9449it [1:09:35, 2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1103
+
1104
+
1105
+
1106
+
1107
+ Training...: 9499it [1:09:55, 2.69it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1108
+
1109
+
1110
+
1111
+
1112
+ Training...: 9549it [1:10:15, 2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1113
+
1114
+
1115
+
1116
+
1117
+
1118
+ Training...: 9602it [1:10:49, 2.24s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1119
+
1120
+
1121
+
1122
+
1123
+
1124
+ Training...: 9654it [1:11:09, 1.39s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1125
+
1126
+
1127
+
1128
+
1129
+
1130
+ Training...: 9704it [1:11:29, 1.39s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1131
+
1132
+
1133
+
1134
+
1135
+ Training...: 9749it [1:11:35, 2.69it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1136
+
1137
+
1138
+
1139
+
1140
+ Training...: 9799it [1:11:55, 2.67it/s]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1141
+
1142
+
1143
+
1144
+
1145
+
1146
+ Training...: 9850it [1:12:29, 4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1147
+
1148
+
1149
+
1150
+
1151
+
1152
+ Training...: 9900it [1:12:49, 4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1153
+
1154
+
1155
+
1156
+
1157
+
1158
+ Training...: 9950it [1:13:09, 4.01s/it]████████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1159
+
1160
+
1161
+
1162
+
1163
+
1164
+ Training...: 10002it [1:13:29, 2.24s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1165
+
1166
+
1167
+
1168
+
1169
+
1170
+ Training...: 10052it [1:13:49, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1171
+
1172
+
1173
+
1174
+
1175
+
1176
+ Training...: 10104it [1:14:09, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1177
+
1178
+
1179
+
1180
+
1181
+
1182
+ Training...: 10154it [1:14:29, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1183
+
1184
+
1185
+
1186
+
1187
+
1188
+ Training...: 10200it [1:14:49, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1189
+
1190
+
1191
+
1192
+
1193
+
1194
+ Training...: 10254it [1:15:09, 1.38s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1195
+
1196
+
1197
+
1198
+
1199
+
1200
+ Training...: 10304it [1:15:29, 1.40s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1201
+
1202
+
1203
+
1204
+
1205
+
1206
+ Training...: 10356it [1:15:49, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1207
+
1208
+
1209
+
1210
+
1211
+
1212
+ Training...: 10406it [1:16:09, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1213
+
1214
+
1215
+
1216
+
1217
+
1218
+ Training...: 10456it [1:16:29, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1219
+
1220
+
1221
+
1222
+
1223
+
1224
+ Training...: 10500it [1:16:49, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1225
+
1226
+
1227
+
1228
+
1229
+
1230
+ Training...: 10550it [1:17:09, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1231
+
1232
+
1233
+
1234
+
1235
+
1236
+ Training...: 10600it [1:17:29, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1237
+
1238
+
1239
+
1240
+
1241
+
1242
+ Training...: 10652it [1:17:49, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1243
+
1244
+
1245
+
1246
+
1247
+
1248
+ Training...: 10704it [1:18:10, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1249
+
1250
+
1251
+
1252
+
1253
+
1254
+ Training...: 10756it [1:18:30, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1255
+
1256
+
1257
+
1258
+
1259
+
1260
+ Training...: 10806it [1:18:50, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1261
+
1262
+
1263
+
1264
+
1265
+
1266
+ Training...: 10850it [1:19:09, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1267
+
1268
+
1269
+
1270
+
1271
+
1272
+ Training...: 10900it [1:19:29, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1273
+
1274
+
1275
+
1276
+
1277
+
1278
+ Training...: 10950it [1:19:49, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1279
+
1280
+
1281
+
1282
+
1283
+
1284
+ Training...: 11002it [1:20:10, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1285
+
1286
+
1287
+
1288
+
1289
+
1290
+ Training...: 11052it [1:20:30, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1291
+
1292
+
1293
+
1294
+
1295
+
1296
+ Training...: 11104it [1:20:50, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1297
+
1298
+
1299
+
1300
+
1301
+
1302
+ Training...: 11154it [1:21:10, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1303
+
1304
+
1305
+
1306
+
1307
+
1308
+ Training...: 11204it [1:21:30, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1309
+
1310
+
1311
+
1312
+
1313
+
1314
+ Training...: 11256it [1:21:50, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1315
+
1316
+
1317
+
1318
+
1319
+
1320
+ Training...: 11302it [1:22:10, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1321
+
1322
+
1323
+
1324
+
1325
+
1326
+ Training...: 11356it [1:22:30, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1327
+
1328
+
1329
+
1330
+
1331
+
1332
+ Training...: 11406it [1:22:50, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1333
+
1334
+
1335
+
1336
+
1337
+
1338
+ Training...: 11458it [1:23:10, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1339
+
1340
+
1341
+
1342
+
1343
+
1344
+ Training...: 11500it [1:23:30, 4.01s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1345
+
1346
+
1347
+
1348
+
1349
+
1350
+ Training...: 11552it [1:23:50, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1351
+
1352
+
1353
+
1354
+
1355
+
1356
+ Training...: 11602it [1:24:10, 2.25s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1357
+
1358
+
1359
+
1360
+
1361
+
1362
+ Training...: 11654it [1:24:30, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1363
+
1364
+
1365
+
1366
+
1367
+
1368
+ Training...: 11704it [1:24:50, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1369
+
1370
+
1371
+
1372
+
1373
+
1374
+ Training...: 11754it [1:25:10, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1375
+
1376
+
1377
+
1378
+
1379
+
1380
+ Training...: 11804it [1:25:30, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1381
+
1382
+
1383
+
1384
+
1385
+
1386
+ Training...: 11856it [1:25:51, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1387
+
1388
+
1389
+
1390
+
1391
+
1392
+ Training...: 11906it [1:26:11, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1393
+
1394
+
1395
+
1396
+
1397
+
1398
+ Training...: 11956it [1:26:31, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1399
+
1400
+
1401
+
1402
+
1403
+
1404
+ Training...: 12008it [1:26:51, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1405
+
1406
+
1407
+
1408
+
1409
+
1410
+ Training...: 12060it [1:27:11, 2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1411
+
1412
+
1413
+
1414
+
1415
+
1416
+ Training...: 12102it [1:27:30, 2.24s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1417
+
1418
+
1419
+
1420
+
1421
+
1422
+ Training...: 12154it [1:27:51, 1.39s/it]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1423
+
1424
+
1425
+
1426
+
1427
+
1428
+ Training...: 12206it [1:28:11, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1429
+
1430
+
1431
+
1432
+
1433
+
1434
+ Training...: 12256it [1:28:31, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1435
+
1436
+
1437
+
1438
+
1439
+
1440
+ Training...: 12308it [1:28:51, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1441
+
1442
+
1443
+
1444
+
1445
+
1446
+ Training...: 12358it [1:29:11, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1447
+
1448
+
1449
+
1450
+
1451
+
1452
+ Training...: 12406it [1:29:31, 1.05it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1453
+
1454
+
1455
+
1456
+
1457
+
1458
+ Training...: 12458it [1:29:51, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1459
+
1460
+
1461
+
1462
+
1463
+
1464
+ Training...: 12508it [1:30:11, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1465
+
1466
+
1467
+
1468
+
1469
+
1470
+ Training...: 12560it [1:30:31, 2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1471
+
1472
+
1473
+
1474
+
1475
+
1476
+ Training...: 12610it [1:30:51, 2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1477
+
1478
+
1479
+
1480
+
1481
+
1482
+ Training...: 12659it [1:31:12, 2.14it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1483
+
1484
+
1485
+
1486
+
1487
+
1488
+ Training...: 12712it [1:31:32, 3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1489
+
1490
+
1491
+
1492
+
1493
+ Training...: 12764it [1:31:52, 4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1494
+
1495
+
1496
+
1497
+
1498
+
1499
+ Training...: 12806it [1:32:11, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1500
+
1501
+
1502
+
1503
+
1504
+
1505
+ Training...: 12856it [1:32:31, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1506
+
1507
+
1508
+
1509
+
1510
+
1511
+ Training...: 12908it [1:32:51, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1512
+
1513
+
1514
+
1515
+
1516
+
1517
+ Training...: 12958it [1:33:12, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1518
+
1519
+
1520
+
1521
+
1522
+
1523
+ Training...: 13008it [1:33:32, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1524
+
1525
+
1526
+
1527
+
1528
+
1529
+ Training...: 13058it [1:33:52, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1530
+
1531
+
1532
+
1533
+
1534
+
1535
+ Training...: 13110it [1:34:12, 2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1536
+
1537
+
1538
+
1539
+
1540
+
1541
+ Training...: 13156it [1:34:32, 1.09it/s]███████���███████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1542
+
1543
+
1544
+
1545
+
1546
+
1547
+ Training...: 13206it [1:34:52, 1.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1548
+
1549
+
1550
+
1551
+
1552
+ Training...: 13249it [1:34:58, 2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1553
+
1554
+
1555
+
1556
+
1557
+
1558
+ Training...: 13299it [1:35:18, 2.70it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1559
+
1560
+
1561
+
1562
+
1563
+
1564
+ Training...: 13349it [1:35:38, 2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1565
+
1566
+
1567
+
1568
+
1569
+
1570
+
1571
+ Training...: 13408it [1:36:12, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1572
+
1573
+
1574
+
1575
+
1576
+
1577
+ Training...: 13460it [1:36:32, 2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1578
+
1579
+
1580
+
1581
+
1582
+
1583
+ Training...: 13508it [1:36:52, 1.59it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1584
+
1585
+
1586
+
1587
+
1588
+
1589
+ Training...: 13562it [1:37:12, 3.10it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1590
+
1591
+
1592
+
1593
+
1594
+
1595
+ Training...: 13612it [1:37:32, 3.05it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1596
+
1597
+
1598
+
1599
+
1600
+ Training...: 13662it [1:37:52, 3.10it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1601
+
1602
+
1603
+
1604
+
1605
+ Training...: 13714it [1:38:12, 4.15it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1606
+
1607
+
1608
+
1609
+
1610
+ Training...: 13760it [1:38:32, 2.19it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1611
+
1612
+
1613
+
1614
+
1615
+ Training...: 13814it [1:38:53, 4.15it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1616
+
1617
+
1618
+
1619
+ Training...: 13849it [1:38:59, 2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1620
+
1621
+
1622
+
1623
+
1624
+ Training...: 13899it [1:39:19, 2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1625
+
1626
+
1627
+
1628
+
1629
+
1630
+ Training...: 13949it [1:39:39, 2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1631
+
1632
+
1633
+
1634
+
1635
+
1636
+
1637
+ Training...: 14010it [1:40:12, 2.25it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1638
+
1639
+
1640
+
1641
+
1642
+
1643
+ Training...: 14060it [1:40:32, 2.26it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1644
+
1645
+
1646
+
1647
+
1648
+
1649
+ Training...: 14112it [1:40:53, 3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1650
+
1651
+
1652
+
1653
+
1654
+
1655
+ Training...: 14162it [1:41:13, 3.09it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1656
+
1657
+
1658
+
1659
+
1660
+
1661
+ Training...: 14212it [1:41:33, 3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1662
+
1663
+
1664
+
1665
+
1666
+ Training...: 14262it [1:41:53, 3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1667
+
1668
+
1669
+
1670
+
1671
+ Training...: 14314it [1:42:13, 4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1672
+
1673
+
1674
+
1675
+
1676
+ Training...: 14364it [1:42:33, 4.17it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1677
+
1678
+
1679
+
1680
+
1681
+ Training...: 14414it [1:42:53, 4.15it/s]████████████████████��██████████████████| 500/500 [00:59<00:00, 7.90it/s]
1682
+
1683
+
1684
+
1685
+
1686
+ Training...: 14464it [1:43:13, 4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1687
+
1688
+
1689
+
1690
+
1691
+ Training...: 14512it [1:43:33, 3.07it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1692
+
1693
+
1694
+
1695
+
1696
+ Training...: 14564it [1:43:53, 4.10it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1697
+
1698
+
1699
+
1700
+ Training...: 14599it [1:43:59, 2.68it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1701
+
1702
+
1703
+
1704
+
1705
+ Training...: 14649it [1:44:19, 2.71it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1706
+
1707
+
1708
+
1709
+
1710
+ Training...: 14699it [1:44:39, 2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1711
+
1712
+
1713
+
1714
+
1715
+
1716
+ Training...: 14762it [1:45:13, 3.11it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1717
+
1718
+
1719
+
1720
+
1721
+ Training...: 14814it [1:45:33, 4.16it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1722
+
1723
+
1724
+
1725
+
1726
+ Training...: 14862it [1:45:53, 2.98it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1727
+
1728
+
1729
+
1730
+ Training...: 14899it [1:45:59, 2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1731
+
1732
+
1733
+
1734
+
1735
+ Training...: 14949it [1:46:19, 2.67it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1736
+
1737
+
1738
+
1739
+
1740
+
1741
+ Training...: 14999it [1:46:39, 2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1742
+ Training...: 14999it [1:46:53, 2.69it/s]███████████████████████████████████████| 500/500 [00:59<00:00, 7.90it/s]
1743
+
1744
+
1745
+
1746
+
1747
+
1748
+
1749
+
1750
+
1751
+
1752
+
1753
+
1754
+
1755
+
1756
+
1757
+
1758
+
1759
+
1760
+
1761
+
1762
+
1763
+
1764
+
1765
+
1766
+
1767
+
1768
+
1769
+
1770
+
1771
+
1772
+
1773
+
1774
+ tcmalloc: large alloc 1354776576 bytes == 0x37b2fe000 @ 0x7fe91a70f680 0x7fe91a72fbdd 0x7fe900cc920d 0x7fe900cd7340 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd2bd3 0x7fe900cd31fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161 0x67e1df
1775
+ tcmalloc: large alloc 2715181056 bytes == 0x3cc702000 @ 0x7fe91a70f680 0x7fe91a72fbdd 0x7fe900cc920d 0x7fe900cd7340 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd6e87 0x7fe900cd2bd3 0x7fe900cd31fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a
1776
+ tcmalloc: large alloc 1530273792 bytes == 0x321dba000 @ 0x7fe91a70f680 0x7fe91a730824 0x5f7b11 0x7fe900cd2c6f 0x7fe900cd31fe 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x5f5956 0x56aadf 0x5f5956 0x56fb87 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fe91a5240b3 0x5f96de
1777
+ [11:53:20] - INFO - absl - Saved checkpoint at checkpoint_240000
1778
+ [11:53:21] - INFO - huggingface_hub.repository - git version 2.25.1
1779
+ git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
wandb/run-20210716_095921-13hxxunp/files/requirements.txt ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.13.0
2
+ aiohttp==3.7.4.post0
3
+ astunparse==1.6.3
4
+ async-timeout==3.0.1
5
+ attrs==21.2.0
6
+ cachetools==4.2.2
7
+ certifi==2021.5.30
8
+ chardet==4.0.0
9
+ charset-normalizer==2.0.1
10
+ chex==0.0.8
11
+ click==8.0.1
12
+ configparser==5.0.2
13
+ cycler==0.10.0
14
+ datasets==1.9.1.dev0
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.0.12
19
+ flatbuffers==1.12
20
+ flax==0.3.4
21
+ fsspec==2021.7.0
22
+ gast==0.4.0
23
+ gitdb==4.0.7
24
+ gitpython==3.1.18
25
+ google-auth-oauthlib==0.4.4
26
+ google-auth==1.32.1
27
+ google-pasta==0.2.0
28
+ grpcio==1.34.1
29
+ h5py==3.1.0
30
+ huggingface-hub==0.0.12
31
+ idna==3.2
32
+ install==1.3.4
33
+ jax==0.2.17
34
+ jaxlib==0.1.68
35
+ joblib==1.0.1
36
+ keras-nightly==2.5.0.dev2021032900
37
+ keras-preprocessing==1.1.2
38
+ kiwisolver==1.3.1
39
+ libtpu-nightly==0.1.dev20210615
40
+ markdown==3.3.4
41
+ matplotlib==3.4.2
42
+ msgpack==1.0.2
43
+ multidict==5.1.0
44
+ multiprocess==0.70.12.2
45
+ numpy==1.19.5
46
+ oauthlib==3.1.1
47
+ opt-einsum==3.3.0
48
+ optax==0.0.9
49
+ packaging==21.0
50
+ pandas==1.3.0
51
+ pathtools==0.1.2
52
+ pillow==8.3.1
53
+ pip==20.0.2
54
+ pkg-resources==0.0.0
55
+ promise==2.3
56
+ protobuf==3.17.3
57
+ psutil==5.8.0
58
+ pyarrow==4.0.1
59
+ pyasn1-modules==0.2.8
60
+ pyasn1==0.4.8
61
+ pyparsing==2.4.7
62
+ python-dateutil==2.8.1
63
+ pytz==2021.1
64
+ pyyaml==5.4.1
65
+ regex==2021.7.6
66
+ requests-oauthlib==1.3.0
67
+ requests==2.26.0
68
+ rsa==4.7.2
69
+ sacremoses==0.0.45
70
+ scipy==1.7.0
71
+ sentry-sdk==1.3.0
72
+ setuptools==44.0.0
73
+ shortuuid==1.0.1
74
+ six==1.15.0
75
+ smmap==4.0.0
76
+ subprocess32==3.5.4
77
+ tensorboard-data-server==0.6.1
78
+ tensorboard-plugin-wit==1.8.0
79
+ tensorboard==2.5.0
80
+ tensorflow-estimator==2.5.0
81
+ tensorflow==2.5.0
82
+ termcolor==1.1.0
83
+ tokenizers==0.10.3
84
+ toolz==0.11.1
85
+ torch==1.9.0
86
+ tqdm==4.61.2
87
+ transformers==4.9.0.dev0
88
+ typing-extensions==3.7.4.3
89
+ urllib3==1.26.6
90
+ wandb==0.10.33
91
+ werkzeug==2.0.1
92
+ wheel==0.36.2
93
+ wrapt==1.12.1
94
+ xxhash==2.0.2
95
+ yarl==1.6.3
wandb/run-20210716_095921-13hxxunp/files/wandb-metadata.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-07-16T09:59:24.007461",
5
+ "startedAt": "2021-07-16T09:59:21.964690",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--push_to_hub",
11
+ "--output_dir=./",
12
+ "--model_type=big_bird",
13
+ "--config_name=./",
14
+ "--tokenizer_name=./",
15
+ "--max_seq_length=4096",
16
+ "--weight_decay=0.0095",
17
+ "--warmup_steps=10000",
18
+ "--overwrite_output_dir",
19
+ "--adam_beta1=0.9",
20
+ "--adam_beta2=0.98",
21
+ "--logging_steps=50",
22
+ "--eval_steps=6000",
23
+ "--num_train_epochs=5",
24
+ "--preprocessing_num_workers=96",
25
+ "--save_steps=15000",
26
+ "--learning_rate=3e-5",
27
+ "--per_device_train_batch_size=1",
28
+ "--per_device_eval_batch_size=1",
29
+ "--save_total_limit=20",
30
+ "--max_eval_samples=4000",
31
+ "--resume_from_checkpoint=./"
32
+ ],
33
+ "state": "running",
34
+ "program": "./run_mlm_flax_no_accum.py",
35
+ "codePath": "run_mlm_flax_no_accum.py",
36
+ "git": {
37
+ "remote": "https://huggingface.co/flax-community/pino-roberta-base",
38
+ "commit": "0e50dbbb7c47cf4a8062023d210e1d841e9cebea"
39
+ },
40
+ "email": null,
41
+ "root": "/home/dat/pino-roberta-base",
42
+ "host": "t1v-n-f5c06ea1-w-0",
43
+ "username": "dat",
44
+ "executable": "/home/dat/pino/bin/python"
45
+ }
wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
1
+ {"training_step": 240000, "learning_rate": 2.606575253594201e-05, "train_loss": 1.9378972053527832, "_runtime": 6829, "_timestamp": 1626436391, "_step": 302, "eval_step": 240000, "eval_accuracy": 0.6196709871292114, "eval_loss": 1.9561585187911987}
wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
wandb/run-20210716_095921-13hxxunp/logs/debug.log ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/logs/debug.log
4
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log
5
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [wandb_init.py:init():419] starting backend
9
+ 2021-07-16 09:59:21,966 INFO MainThread:782479 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-16 09:59:22,014 INFO MainThread:782479 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-16 09:59:22,062 INFO MainThread:782479 [backend.py:ensure_launched():139] started backend process with pid: 783720
12
+ 2021-07-16 09:59:22,065 INFO MainThread:782479 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-16 09:59:22,068 INFO MainThread:782479 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-16 09:59:22,069 INFO MainThread:782479 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-16 09:59:22,707 INFO MainThread:782479 [wandb_init.py:init():496] got version response upgrade_message: "wandb version 0.11.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade"
16
+
17
+ 2021-07-16 09:59:22,707 INFO MainThread:782479 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
18
+ 2021-07-16 09:59:22,884 INFO MainThread:782479 [wandb_init.py:init():529] starting run threads in backend
19
+ 2021-07-16 09:59:24,052 INFO MainThread:782479 [wandb_run.py:_console_start():1623] atexit reg
20
+ 2021-07-16 09:59:24,053 INFO MainThread:782479 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
21
+ 2021-07-16 09:59:24,054 INFO MainThread:782479 [wandb_run.py:_redirect():1502] Redirecting console.
22
+ 2021-07-16 09:59:24,055 INFO MainThread:782479 [wandb_run.py:_redirect():1558] Redirects installed.
23
+ 2021-07-16 09:59:24,056 INFO MainThread:782479 [wandb_init.py:init():554] run started, returning control to user process
24
+ 2021-07-16 09:59:24,061 INFO MainThread:782479 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul16_09-59-13_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 15000, 'save_total_limit': 20, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 6000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
25
+ 2021-07-16 09:59:24,063 INFO MainThread:782479 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
26
+ 2021-07-16 09:59:24,065 INFO MainThread:782479 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 4000}
wandb/run-20210716_095921-13hxxunp/run-13hxxunp.wandb ADDED
Binary file (426 kB). View file