Saving weights and logs of step 5000

events.out.tfevents.1625843003.t1v-n-528d9406-w-0.250031.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65f1a7d1164ca3b62fcc3dea900488379e35e8dbc651d2d586a093e695813bb7
+size 735138

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68d1eca2a3ae4fc5ba1ba43e64ef83cd58cf8add1c0aee8478bf5651b0bcd130
+oid sha256:c82df389e510b831d9a5c33f348a258ef105f78170ccc3efd78ab838d40a038b
 size 1419302302

run_pretraining.sh

@@ -20,8 +20,8 @@ export WANDB_LOG_MODEL="true"
     --num_train_epochs="20" \
     --dataloader_num_workers="64" \
     --preprocessing_num_workers="64" \
-    --save_steps="10" \
-    --eval_steps="10" \
-    --max_train_samples="10000" \
-    --max_eval_samples="1000" \
+    --logging_steps="5000" \
+    --save_steps="5000" \
+    --eval_steps="5000" \
+    --validation_split_percentage="2" \
     --push_to_hub

wandb/debug-internal.log

@@ -1 +1 @@
-run-20210709_141445-2k8cnty2/logs/debug-internal.log
+run-20210709_144100-2k1kyrq2/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20210709_141445-2k8cnty2/logs/debug.log
+run-20210709_144100-2k1kyrq2/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20210709_141445-2k8cnty2
+run-20210709_144100-2k1kyrq2

wandb/run-20210709_141445-2k8cnty2/files/config.yaml

@@ -20,6 +20,10 @@ _wandb:
       - 1
       - 3
       - 11
+      2:
+      - 1
+      - 3
+      - 11
       4: 3.8.10
       5: 0.10.33
       6: 4.9.0.dev0

wandb/run-20210709_141445-2k8cnty2/files/output.log

@@ -528,3 +528,17 @@ Training...:  92%|████████████████████
 Evaluating...:   0%|                                                                                                                                                                                                           | 0/5 [00:00<?, ?it/s]
 
 
+Model weights saved in /home/cahya/Work/flax-community/gpt2-medium-indonesian/flax_model.msgpack██████████████████████████████████████████████████████████████████████████████████████████████████████               | 48/52 [05:47<00:06,  1.74s/it]
+Model pushed to the hub in this commit: https://huggingface.co/flax-community/gpt2-medium-indonesian/commit/9b9473462e797993d7ca4cfb92daa71c0bc81e9d
+Step... (100 | Eval Loss: 7.69423770904541 | Eval Perplexity: 2195.65944089946):  10%|████████████▌                                                                                                                | 2/20 [14:54<2:13:01, 443.41s/it]
+
+Training...:  12%|██████████████████████▌                                                                                                                                                                             | 6/52 [00:01<00:10,  4.36it/s]
+  File "./run_clm_flax.py", line 655, in <module>
+    main()
+  File "./run_clm_flax.py", line 620, in main
+    batch = next(eval_loader)
+  File "./run_clm_flax.py", line 193, in data_loader
+    for idx in batch_idx:
+  File "/home/cahya/miniconda3/envs/jax/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1264, in __iter__
+    return self._value.__iter__()
+KeyboardInterrupt

wandb/run-20210709_141445-2k8cnty2/files/wandb-summary.json

@@ -1 +1 @@
-{"global_step": 90, "_timestamp": 1625840867.513841, "eval_loss": 7.724389553070068, "eval_perplexity": 2262.870849609375, "_step": 8}
+{"global_step": 100, "_timestamp": 1625840946.414606, "eval_loss": 7.69423770904541, "eval_perplexity": 2195.659423828125, "_step": 9}

wandb/run-20210709_141445-2k8cnty2/logs/debug-internal.log

@@ -330,3 +330,139 @@
 2021-07-09 14:29:24,657 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
 2021-07-09 14:29:28,577 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-09 14:29:28,578 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:29:43,713 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:29:43,713 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:29:47,814 DEBUG   SenderThread:246776 [sender.py:send():179] send: stats
+2021-07-09 14:29:58,843 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:29:58,844 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:30:13,979 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:30:13,980 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:30:17,889 DEBUG   SenderThread:246776 [sender.py:send():179] send: stats
+2021-07-09 14:30:24,683 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:26,684 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:28,685 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:29,111 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:30:29,111 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:30:30,097 WARNING MainThread:246776 [internal.py:wandb_internal():147] Internal process interrupt: 1
+2021-07-09 14:30:30,685 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:34,687 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:36,622 DEBUG   SenderThread:246776 [sender.py:send():179] send: telemetry
+2021-07-09 14:30:36,622 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:36,622 DEBUG   SenderThread:246776 [sender.py:send():179] send: exit
+2021-07-09 14:30:36,622 INFO    SenderThread:246776 [sender.py:send_exit():287] handling exit code: 255
+2021-07-09 14:30:36,623 INFO    SenderThread:246776 [sender.py:send_exit():295] send defer
+2021-07-09 14:30:36,623 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:36,624 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:36,624 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 0
+2021-07-09 14:30:36,624 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:36,624 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 0
+2021-07-09 14:30:36,624 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 1
+2021-07-09 14:30:36,624 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:36,624 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 1
+2021-07-09 14:30:36,631 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:36,631 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 1
+2021-07-09 14:30:36,631 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 2
+2021-07-09 14:30:36,631 DEBUG   SenderThread:246776 [sender.py:send():179] send: stats
+2021-07-09 14:30:36,632 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:36,632 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 2
+2021-07-09 14:30:36,688 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:42,961 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:42,961 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:42,961 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 2
+2021-07-09 14:30:42,961 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:42,961 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 3
+2021-07-09 14:30:42,962 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:42,962 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:42,963 DEBUG   SenderThread:246776 [sender.py:send():179] send: history
+2021-07-09 14:30:42,963 DEBUG   SenderThread:246776 [sender.py:send():179] send: summary
+2021-07-09 14:30:42,963 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:42,964 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 3
+2021-07-09 14:30:42,964 INFO    SenderThread:246776 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 14:30:42,964 DEBUG   SenderThread:246776 [sender.py:send():179] send: summary
+2021-07-09 14:30:42,965 INFO    SenderThread:246776 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 14:30:42,965 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:42,965 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 3
+2021-07-09 14:30:42,965 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 4
+2021-07-09 14:30:42,965 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:42,965 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 4
+2021-07-09 14:30:42,965 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:42,966 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 4
+2021-07-09 14:30:43,065 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:43,170 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 5
+2021-07-09 14:30:43,170 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:43,171 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:43,171 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 5
+2021-07-09 14:30:43,171 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:43,171 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 5
+2021-07-09 14:30:43,171 INFO    SenderThread:246776 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-09 14:30:43,272 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:43,690 INFO    SenderThread:246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/config.yaml
+2021-07-09 14:30:43,691 INFO    SenderThread:246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/wandb-summary.json
+2021-07-09 14:30:43,691 INFO    SenderThread:246776 [dir_watcher.py:finish():312] scan: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files
+2021-07-09 14:30:43,691 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/conda-environment.yaml conda-environment.yaml
+2021-07-09 14:30:43,691 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/requirements.txt requirements.txt
+2021-07-09 14:30:43,691 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log output.log
+2021-07-09 14:30:43,692 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/wandb-metadata.json wandb-metadata.json
+2021-07-09 14:30:43,692 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/config.yaml config.yaml
+2021-07-09 14:30:43,695 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/events.out.tfevents.1625840127.t1v-n-528d9406-w-0.245719.3.v2 events.out.tfevents.1625840127.t1v-n-528d9406-w-0.245719.3.v2
+2021-07-09 14:30:43,695 INFO    SenderThread:246776 [dir_watcher.py:finish():318] scan save: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/wandb-summary.json wandb-summary.json
+2021-07-09 14:30:43,695 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 6
+2021-07-09 14:30:43,695 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:43,696 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:43,705 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 6
+2021-07-09 14:30:43,705 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:43,705 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 6
+2021-07-09 14:30:43,705 INFO    SenderThread:246776 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 14:30:43,798 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:43,798 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:43,900 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:43,900 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,002 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,002 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,104 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,104 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,144 INFO    Thread-24 :246776 [upload_job.py:push():137] Uploaded file /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/config.yaml
+2021-07-09 14:30:44,145 INFO    Thread-22 :246776 [upload_job.py:push():137] Uploaded file /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/requirements.txt
+2021-07-09 14:30:44,154 INFO    Thread-21 :246776 [upload_job.py:push():137] Uploaded file /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/conda-environment.yaml
+2021-07-09 14:30:44,183 INFO    Thread-26 :246776 [upload_job.py:push():137] Uploaded file /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/wandb-summary.json
+2021-07-09 14:30:44,206 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,206 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,232 INFO    Thread-25 :246776 [upload_job.py:push():137] Uploaded file /tmp/tmp4f0a6i2mwandb/i1mxdnz1-events.out.tfevents.1625840127.t1v-n-528d9406-w-0.245719.3.v2
+2021-07-09 14:30:44,307 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,308 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,359 INFO    Thread-23 :246776 [upload_job.py:push():137] Uploaded file /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:30:44,409 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,409 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,511 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,511 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,559 INFO    Thread-7  :246776 [sender.py:transition_state():308] send defer: 7
+2021-07-09 14:30:44,560 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:44,560 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 7
+2021-07-09 14:30:44,560 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:44,560 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 7
+2021-07-09 14:30:44,612 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,831 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 8
+2021-07-09 14:30:44,831 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,832 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:44,832 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 8
+2021-07-09 14:30:44,832 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:44,832 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 8
+2021-07-09 14:30:44,832 INFO    SenderThread:246776 [sender.py:transition_state():308] send defer: 9
+2021-07-09 14:30:44,833 DEBUG   SenderThread:246776 [sender.py:send():179] send: final
+2021-07-09 14:30:44,833 DEBUG   SenderThread:246776 [sender.py:send():179] send: footer
+2021-07-09 14:30:44,833 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: defer
+2021-07-09 14:30:44,833 INFO    HandlerThread:246776 [handler.py:handle_request_defer():141] handle defer: 9
+2021-07-09 14:30:44,833 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: defer
+2021-07-09 14:30:44,833 INFO    SenderThread:246776 [sender.py:send_request_defer():304] handle sender defer: 9
+2021-07-09 14:30:44,933 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: poll_exit
+2021-07-09 14:30:44,933 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: poll_exit
+2021-07-09 14:30:44,933 INFO    SenderThread:246776 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 14:30:44,935 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: get_summary
+2021-07-09 14:30:44,936 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: sampled_history
+2021-07-09 14:30:44,937 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: shutdown
+2021-07-09 14:30:44,937 INFO    HandlerThread:246776 [handler.py:finish():638] shutting down handler
+2021-07-09 14:30:45,833 INFO    WriterThread:246776 [datastore.py:close():288] close: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/run-2k8cnty2.wandb
+2021-07-09 14:30:45,934 INFO    SenderThread:246776 [sender.py:finish():945] shutting down sender
+2021-07-09 14:30:45,934 INFO    SenderThread:246776 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-09 14:30:45,934 INFO    SenderThread:246776 [file_pusher.py:join():182] waiting for file pusher
+2021-07-09 14:30:45,936 INFO    MainThread:246776 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_141445-2k8cnty2/logs/debug.log

@@ -27,3 +27,138 @@ config: {}
 2021-07-09 14:14:49,958 INFO    MainThread:245719 [wandb_config.py:__setitem__():141] config set test_log = 12345 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7fab4d59abb0>>
 2021-07-09 14:14:49,958 INFO    MainThread:245719 [wandb_run.py:_config_callback():872] config_cb test_log 12345 None
 2021-07-09 14:15:27,424 INFO    MainThread:245719 [wandb_run.py:_tensorboard_callback():943] tensorboard callback: /home/cahya/Work/flax-community/gpt2-medium-indonesian, None
+2021-07-09 14:30:34,276 INFO    MainThread:245719 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
+2021-07-09 14:30:34,276 INFO    MainThread:245719 [wandb_run.py:_restore():1565] restore
+2021-07-09 14:30:36,624 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 3022
+  total_bytes: 3022
+}
+
+2021-07-09 14:30:41,726 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: None
+2021-07-09 14:30:42,963 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 3022
+  total_bytes: 3022
+}
+
+2021-07-09 14:30:43,171 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 1
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 3022
+  total_bytes: 3022
+}
+
+2021-07-09 14:30:43,697 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 5
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 3022
+  total_bytes: 100895
+}
+
+2021-07-09 14:30:43,799 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1722
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:43,901 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,003 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,104 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,206 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,308 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,410 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,511 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,832 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:44,934 INFO    MainThread:245719 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 6
+  other_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 101169
+  total_bytes: 101169
+}
+
+2021-07-09 14:30:46,219 INFO    MainThread:245719 [wandb_run.py:_show_summary():1870] rendering summary
+2021-07-09 14:30:46,220 INFO    MainThread:245719 [wandb_run.py:_show_history():1908] rendering history
+2021-07-09 14:30:46,220 INFO    MainThread:245719 [wandb_run.py:_show_files():1937] logging synced files

wandb/run-20210709_144100-2k1kyrq2/files/conda-environment.yaml

@@ -0,0 +1,187 @@
+name: jax
+channels:
+  - pytorch
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=4.5=1_gnu
+  - anyio=3.2.1=py38h578d9bd_0
+  - argon2-cffi=20.1.0=py38h497a2fe_2
+  - async_generator=1.10=py_0
+  - attrs=21.2.0=pyhd8ed1ab_0
+  - babel=2.9.1=pyh44b312d_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=py_2
+  - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
+  - blas=1.0=mkl
+  - bleach=3.3.0=pyh44b312d_0
+  - brotlipy=0.7.0=py38h497a2fe_1001
+  - ca-certificates=2021.5.30=ha878542_0
+  - certifi=2021.5.30=py38h578d9bd_0
+  - cffi=1.14.5=py38ha65f79e_0
+  - chardet=4.0.0=py38h578d9bd_1
+  - cryptography=3.4.7=py38ha5dfef3_0
+  - cudatoolkit=10.2.89=hfd86e86_1
+  - debugpy=1.3.0=py38h709712a_0
+  - decorator=5.0.9=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - entrypoints=0.3=pyhd8ed1ab_1003
+  - idna=2.10=pyh9f0ad1d_0
+  - importlib-metadata=4.6.1=py38h578d9bd_0
+  - intel-openmp=2021.2.0=h06a4308_610
+  - ipykernel=6.0.1=py38hd0cf306_0
+  - ipython=7.25.0=py38hd0cf306_1
+  - ipython_genutils=0.2.0=py_1
+  - jedi=0.18.0=py38h578d9bd_2
+  - jinja2=3.0.1=pyhd8ed1ab_0
+  - json5=0.9.5=pyh9f0ad1d_0
+  - jsonschema=3.2.0=pyhd8ed1ab_3
+  - jupyter_client=6.1.12=pyhd8ed1ab_0
+  - jupyter_core=4.7.1=py38h578d9bd_0
+  - jupyter_server=1.9.0=pyhd8ed1ab_0
+  - jupyterlab=3.0.16=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.1.2=pyh9f0ad1d_0
+  - jupyterlab_server=2.6.0=pyhd8ed1ab_0
+  - ld_impl_linux-64=2.35.1=h7274673_9
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=9.3.0=h5101ec6_17
+  - libgomp=9.3.0=h5101ec6_17
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=9.3.0=hd4cf53a_17
+  - libuv=1.40.0=h7b6447c_0
+  - markupsafe=2.0.1=py38h497a2fe_0
+  - matplotlib-inline=0.1.2=pyhd8ed1ab_2
+  - mistune=0.8.4=py38h497a2fe_1004
+  - mkl=2021.2.0=h06a4308_296
+  - nbclassic=0.3.1=pyhd8ed1ab_1
+  - nbclient=0.5.3=pyhd8ed1ab_0
+  - nbconvert=6.1.0=py38h578d9bd_0
+  - nbformat=5.1.3=pyhd8ed1ab_0
+  - ncurses=6.2=he6710b0_1
+  - nest-asyncio=1.5.1=pyhd8ed1ab_0
+  - ninja=1.10.2=hff7bd54_1
+  - notebook=6.4.0=pyha770c72_0
+  - openssl=1.1.1k=h7f98852_0
+  - pandoc=2.14.0.3=h7f98852_0
+  - pandocfilters=1.4.2=py_1
+  - parso=0.8.2=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh9f0ad1d_2
+  - pickleshare=0.7.5=py_1003
+  - pip=21.1.3=py38h06a4308_0
+  - prometheus_client=0.11.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.19=pyha770c72_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pycparser=2.20=pyh9f0ad1d_2
+  - pygments=2.9.0=pyhd8ed1ab_0
+  - pyopenssl=20.0.1=pyhd8ed1ab_0
+  - pyparsing=2.4.7=pyh9f0ad1d_0
+  - pyrsistent=0.17.3=py38h497a2fe_2
+  - pysocks=1.7.1=py38h578d9bd_3
+  - python=3.8.10=h12debd9_8
+  - python-dateutil=2.8.1=py_0
+  - python_abi=3.8=2_cp38
+  - pytorch=1.9.0=py3.8_cuda10.2_cudnn7.6.5_0
+  - pytz=2021.1=pyhd8ed1ab_0
+  - pyzmq=22.1.0=py38h2035c66_0
+  - readline=8.1=h27cfd23_0
+  - requests=2.25.1=pyhd3deb0d_0
+  - requests-unixsocket=0.2.0=py_0
+  - send2trash=1.7.1=pyhd8ed1ab_0
+  - setuptools=52.0.0=py38h06a4308_0
+  - sniffio=1.2.0=py38h578d9bd_1
+  - sqlite=3.36.0=hc218d9a_0
+  - terminado=0.10.1=py38h578d9bd_0
+  - testpath=0.5.0=pyhd8ed1ab_0
+  - tk=8.6.10=hbc83047_0
+  - tornado=6.1=py38h497a2fe_1
+  - traitlets=5.0.5=py_0
+  - typing_extensions=3.10.0.0=pyha770c72_0
+  - urllib3=1.26.6=pyhd8ed1ab_0
+  - wcwidth=0.2.5=pyh9f0ad1d_2
+  - webencodings=0.5.1=py_1
+  - websocket-client=0.57.0=py38h578d9bd_4
+  - wheel=0.36.2=pyhd3eb1b0_0
+  - xz=5.2.5=h7b6447c_0
+  - zeromq=4.3.4=h9c3ff4c_0
+  - zipp=3.5.0=pyhd8ed1ab_0
+  - zlib=1.2.11=h7b6447c_3
+  - pip:
+    - absl-py==0.13.0
+    - aiohttp==3.7.4.post0
+    - astunparse==1.6.3
+    - async-timeout==3.0.1
+    - cachetools==4.2.2
+    - chex==0.0.8
+    - click==8.0.1
+    - configparser==5.0.2
+    - cycler==0.10.0
+    - dill==0.3.4
+    - dm-tree==0.1.6
+    - docker-pycreds==0.4.0
+    - filelock==3.0.12
+    - flatbuffers==1.12
+    - flax==0.3.4
+    - fsspec==2021.6.1
+    - gast==0.4.0
+    - gitdb==4.0.7
+    - gitpython==3.1.18
+    - google-auth==1.32.1
+    - google-auth-oauthlib==0.4.4
+    - google-pasta==0.2.0
+    - grpcio==1.34.1
+    - h5py==3.1.0
+    - huggingface-hub==0.0.12
+    - jax==0.2.16
+    - jaxlib==0.1.68
+    - joblib==1.0.1
+    - keras-nightly==2.5.0.dev2021032900
+    - keras-preprocessing==1.1.2
+    - kiwisolver==1.3.1
+    - libtpu-nightly==0.1.dev20210615
+    - markdown==3.3.4
+    - matplotlib==3.4.2
+    - msgpack==1.0.2
+    - multidict==5.1.0
+    - multiprocess==0.70.12.2
+    - numpy==1.19.5
+    - oauthlib==3.1.1
+    - opt-einsum==3.3.0
+    - optax==0.0.9
+    - packaging==20.9
+    - pandas==1.2.5
+    - pathtools==0.1.2
+    - pillow==8.3.0
+    - promise==2.3
+    - protobuf==3.17.3
+    - psutil==5.8.0
+    - pyarrow==4.0.1
+    - pyasn1==0.4.8
+    - pyasn1-modules==0.2.8
+    - pyyaml==5.4.1
+    - regex==2021.7.1
+    - requests-oauthlib==1.3.0
+    - rsa==4.7.2
+    - sacremoses==0.0.45
+    - scipy==1.7.0
+    - sentry-sdk==1.1.0
+    - shortuuid==1.0.1
+    - six==1.15.0
+    - smmap==4.0.0
+    - subprocess32==3.5.4
+    - tensorboard==2.5.0
+    - tensorboard-data-server==0.6.1
+    - tensorboard-plugin-wit==1.8.0
+    - tensorflow==2.5.0
+    - tensorflow-estimator==2.5.0
+    - termcolor==1.1.0
+    - tokenizers==0.10.3
+    - toolz==0.11.1
+    - tqdm==4.61.1
+    - typing-extensions==3.7.4.3
+    - wandb==0.10.33
+    - werkzeug==2.0.1
+    - wrapt==1.12.1
+    - xxhash==2.0.2
+    - yarl==1.6.3
+prefix: /home/cahya/miniconda3/envs/jax

wandb/run-20210709_144100-2k1kyrq2/files/config.yaml

@@ -0,0 +1,305 @@
+wandb_version: 1
+
+__cached__setup_devices:
+  desc: null
+  value: cpu
+_n_gpu:
+  desc: null
+  value: 0
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 1
+      - 3
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.98
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+block_size:
+  desc: null
+  value: 512
+cache_dir:
+  desc: null
+  value: null
+config_name:
+  desc: null
+  value: /home/cahya/Work/flax-community/gpt2-medium-indonesian
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 64
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataset_config_name:
+  desc: null
+  value: unshuffled_deduplicated_id
+dataset_name:
+  desc: null
+  value: oscar
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+debug:
+  desc: null
+  value: []
+deepspeed:
+  desc: null
+  value: null
+disable_tqdm:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: true
+dtype:
+  desc: null
+  value: float32
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_steps:
+  desc: null
+  value: 5000
+evaluation_strategy:
+  desc: null
+  value: IntervalStrategy.NO
+fp16:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+greater_is_better:
+  desc: null
+  value: null
+group_by_length:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 0.0024
+length_column_name:
+  desc: null
+  value: length
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: /home/cahya/Work/flax-community/gpt2-medium-indonesian/runs/Jul09_14-41-04_t1v-n-528d9406-w-0
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 5000
+logging_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+lr_scheduler_type:
+  desc: null
+  value: SchedulerType.LINEAR
+max_eval_samples:
+  desc: null
+  value: null
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_steps:
+  desc: null
+  value: -1
+max_train_samples:
+  desc: null
+  value: null
+metric_for_best_model:
+  desc: null
+  value: null
+model_name_or_path:
+  desc: null
+  value: null
+model_type:
+  desc: null
+  value: gpt2
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+num_train_epochs:
+  desc: null
+  value: 20.0
+output_dir:
+  desc: null
+  value: /home/cahya/Work/flax-community/gpt2-medium-indonesian
+overwrite_cache:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 24
+per_device_train_batch_size:
+  desc: null
+  value: 24
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+prediction_loss_only:
+  desc: null
+  value: false
+preprocessing_num_workers:
+  desc: null
+  value: 64
+push_to_hub:
+  desc: null
+  value: true
+push_to_hub_model_id:
+  desc: null
+  value: gpt2-medium-indonesian
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: null
+remove_unused_columns:
+  desc: null
+  value: true
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+resume_from_checkpoint:
+  desc: null
+  value: null
+run_name:
+  desc: null
+  value: /home/cahya/Work/flax-community/gpt2-medium-indonesian
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 5000
+save_strategy:
+  desc: null
+  value: IntervalStrategy.STEPS
+save_total_limit:
+  desc: null
+  value: null
+seed:
+  desc: null
+  value: 42
+sharded_ddp:
+  desc: null
+  value: []
+skip_memory_metrics:
+  desc: null
+  value: true
+test_log:
+  desc: null
+  value: 12345
+tokenizer_name:
+  desc: null
+  value: /home/cahya/Work/flax-community/gpt2-medium-indonesian
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: null
+train_file:
+  desc: null
+  value: null
+use_fast_tokenizer:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+validation_file:
+  desc: null
+  value: null
+validation_split_percentage:
+  desc: null
+  value: 2
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 1000
+weight_decay:
+  desc: null
+  value: 0.01

wandb/run-20210709_144100-2k1kyrq2/files/events.out.tfevents.1625843003.t1v-n-528d9406-w-0.250031.3.v2

@@ -0,0 +1 @@
+/home/cahya/Work/flax-community/gpt2-medium-indonesian/events.out.tfevents.1625843003.t1v-n-528d9406-w-0.250031.3.v2

wandb/run-20210709_144100-2k1kyrq2/files/requirements.txt

@@ -0,0 +1,153 @@
+absl-py==0.13.0
+aiohttp==3.7.4.post0
+anyio==3.2.1
+argon2-cffi==20.1.0
+astunparse==1.6.3
+async-generator==1.10
+async-timeout==3.0.1
+attrs==21.2.0
+babel==2.9.1
+backcall==0.2.0
+backports.functools-lru-cache==1.6.4
+bleach==3.3.0
+brotlipy==0.7.0
+cachetools==4.2.2
+certifi==2021.5.30
+cffi==1.14.5
+chardet==4.0.0
+chex==0.0.8
+click==8.0.1
+configparser==5.0.2
+cryptography==3.4.7
+cycler==0.10.0
+datasets==1.8.1.dev0
+debugpy==1.3.0
+decorator==5.0.9
+defusedxml==0.7.1
+dill==0.3.4
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+entrypoints==0.3
+filelock==3.0.12
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.6.1
+gast==0.4.0
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.4
+google-auth==1.32.1
+google-pasta==0.2.0
+grpcio==1.34.1
+h5py==3.1.0
+huggingface-hub==0.0.12
+idna==2.10
+importlib-metadata==4.6.1
+ipykernel==6.0.1
+ipython-genutils==0.2.0
+ipython==7.25.0
+jax==0.2.16
+jaxlib==0.1.68
+jedi==0.18.0
+jinja2==3.0.1
+joblib==1.0.1
+json5==0.9.5
+jsonschema==3.2.0
+jupyter-client==6.1.12
+jupyter-core==4.7.1
+jupyter-server==1.9.0
+jupyterlab-pygments==0.1.2
+jupyterlab-server==2.6.0
+jupyterlab==3.0.16
+keras-nightly==2.5.0.dev2021032900
+keras-preprocessing==1.1.2
+kiwisolver==1.3.1
+libtpu-nightly==0.1.dev20210615
+markdown==3.3.4
+markupsafe==2.0.1
+matplotlib-inline==0.1.2
+matplotlib==3.4.2
+mistune==0.8.4
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.12.2
+nbclassic==0.3.1
+nbclient==0.5.3
+nbconvert==6.1.0
+nbformat==5.1.3
+nest-asyncio==1.5.1
+notebook==6.4.0
+numpy==1.19.5
+oauthlib==3.1.1
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==21.0
+pandas==1.2.5
+pandocfilters==1.4.2
+parso==0.8.2
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.3.0
+pip==21.1.3
+prometheus-client==0.11.0
+promise==2.3
+prompt-toolkit==3.0.19
+protobuf==3.17.3
+psutil==5.8.0
+ptyprocess==0.7.0
+pyarrow==4.0.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pycparser==2.20
+pygments==2.9.0
+pyopenssl==20.0.1
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pysocks==1.7.1
+python-dateutil==2.8.1
+pytz==2021.1
+pyyaml==5.4.1
+pyzmq==22.1.0
+regex==2021.7.1
+requests-oauthlib==1.3.0
+requests-unixsocket==0.2.0
+requests==2.25.1
+rsa==4.7.2
+sacremoses==0.0.45
+scipy==1.7.0
+send2trash==1.7.1
+sentry-sdk==1.1.0
+setuptools==52.0.0.post20210125
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+sniffio==1.2.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.5.0
+tensorflow==2.5.0
+termcolor==1.1.0
+terminado==0.10.1
+testpath==0.5.0
+tokenizers==0.10.3
+toolz==0.11.1
+torch==1.9.0
+tornado==6.1
+tqdm==4.61.1
+traitlets==5.0.5
+transformers==4.9.0.dev0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.10.33
+wcwidth==0.2.5
+webencodings==0.5.1
+websocket-client==0.57.0
+werkzeug==2.0.1
+wheel==0.36.2
+wrapt==1.12.1
+xxhash==2.0.2
+yarl==1.6.3
+zipp==3.5.0

wandb/run-20210709_144100-2k1kyrq2/files/wandb-metadata.json

@@ -0,0 +1,48 @@
+{
+    "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.17",
+    "python": "3.8.10",
+    "heartbeatAt": "2021-07-09T14:41:02.325218",
+    "startedAt": "2021-07-09T14:41:00.270917",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=/home/cahya/Work/flax-community/gpt2-medium-indonesian",
+        "--model_type=gpt2",
+        "--config_name=/home/cahya/Work/flax-community/gpt2-medium-indonesian",
+        "--tokenizer_name=/home/cahya/Work/flax-community/gpt2-medium-indonesian",
+        "--dataset_name=oscar",
+        "--dataset_config_name=unshuffled_deduplicated_id",
+        "--do_train",
+        "--do_eval",
+        "--block_size=512",
+        "--per_device_train_batch_size=24",
+        "--per_device_eval_batch_size=24",
+        "--learning_rate=0.0024",
+        "--warmup_steps=1000",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--weight_decay=0.01",
+        "--overwrite_output_dir",
+        "--num_train_epochs=20",
+        "--dataloader_num_workers=64",
+        "--preprocessing_num_workers=64",
+        "--logging_steps=5000",
+        "--save_steps=5000",
+        "--eval_steps=5000",
+        "--validation_split_percentage=2",
+        "--push_to_hub"
+    ],
+    "state": "running",
+    "program": "./run_clm_flax.py",
+    "codePath": "run_clm_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/flax-community/gpt2-medium-indonesian",
+        "commit": "9b9473462e797993d7ca4cfb92daa71c0bc81e9d"
+    },
+    "email": "cahya.wirawan@gmail.com",
+    "root": "/home/cahya/Work/flax-community/gpt2-medium-indonesian",
+    "host": "t1v-n-528d9406-w-0",
+    "username": "cahya",
+    "executable": "/home/cahya/miniconda3/envs/jax/bin/python"
+}

wandb/run-20210709_144100-2k1kyrq2/files/wandb-summary.json

@@ -0,0 +1 @@
+{"global_step": 5000, "_timestamp": 1625848638.071834, "train_time": 5622.2197265625, "train_learning_rate": 0.00238410709425807, "_step": 9997, "train_loss": 3.5673484802246094}

wandb/run-20210709_144100-2k1kyrq2/logs/debug.log

@@ -0,0 +1,29 @@
+2021-07-09 14:41:00,272 INFO    MainThread:250031 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_LOG_MODEL
+2021-07-09 14:41:00,272 INFO    MainThread:250031 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-gpt2-indonesian', 'entity': 'wandb'}
+2021-07-09 14:41:00,272 INFO    MainThread:250031 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 14:41:00,272 INFO    MainThread:250031 [wandb_init.py:_log_setup():337] Logging user logs to /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_144100-2k1kyrq2/logs/debug.log
+2021-07-09 14:41:00,272 INFO    MainThread:250031 [wandb_init.py:_log_setup():338] Logging internal logs to /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_144100-2k1kyrq2/logs/debug-internal.log
+2021-07-09 14:41:00,272 INFO    MainThread:250031 [wandb_init.py:init():370] calling init triggers
+2021-07-09 14:41:00,273 INFO    MainThread:250031 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 14:41:00,273 INFO    MainThread:250031 [wandb_init.py:init():419] starting backend
+2021-07-09 14:41:00,273 INFO    MainThread:250031 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 14:41:00,314 INFO    MainThread:250031 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 14:41:00,353 INFO    MainThread:250031 [backend.py:ensure_launched():139] started backend process with pid: 251111
+2021-07-09 14:41:00,355 INFO    MainThread:250031 [wandb_init.py:init():424] backend started and connected
+2021-07-09 14:41:00,357 INFO    MainThread:250031 [wandb_init.py:init():472] updated telemetry
+2021-07-09 14:41:00,358 INFO    MainThread:250031 [wandb_init.py:init():491] communicating current version
+2021-07-09 14:41:01,006 INFO    MainThread:250031 [wandb_init.py:init():496] got version response 
+2021-07-09 14:41:01,006 INFO    MainThread:250031 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 14:41:01,184 INFO    MainThread:250031 [wandb_init.py:init():529] starting run threads in backend
+2021-07-09 14:41:04,780 INFO    MainThread:250031 [wandb_run.py:_console_start():1623] atexit reg
+2021-07-09 14:41:04,781 INFO    MainThread:250031 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
+2021-07-09 14:41:04,782 INFO    MainThread:250031 [wandb_run.py:_redirect():1502] Redirecting console.
+2021-07-09 14:41:04,783 INFO    MainThread:250031 [wandb_run.py:_redirect():1558] Redirects installed.
+2021-07-09 14:41:04,784 INFO    MainThread:250031 [wandb_init.py:init():554] run started, returning control to user process
+2021-07-09 14:41:04,793 INFO    MainThread:250031 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': '/home/cahya/Work/flax-community/gpt2-medium-indonesian', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 24, 'per_device_eval_batch_size': 24, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0024, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 20.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': '/home/cahya/Work/flax-community/gpt2-medium-indonesian/runs/Jul09_14-41-04_t1v-n-528d9406-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 5000, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 5000, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 64, 'past_index': -1, 'run_name': '/home/cahya/Work/flax-community/gpt2-medium-indonesian', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': 'gpt2-medium-indonesian', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
+2021-07-09 14:41:04,794 INFO    MainThread:250031 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'gpt2', 'config_name': '/home/cahya/Work/flax-community/gpt2-medium-indonesian', 'tokenizer_name': '/home/cahya/Work/flax-community/gpt2-medium-indonesian', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
+2021-07-09 14:41:04,794 INFO    MainThread:250031 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_id', 'train_file': None, 'validation_file': None, 'max_train_samples': None, 'max_eval_samples': None, 'overwrite_cache': False, 'validation_split_percentage': 2, 'block_size': 512, 'preprocessing_num_workers': 64}
+2021-07-09 14:41:04,795 INFO    MainThread:250031 [wandb_config.py:__setitem__():141] config set test_log = 12345 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f3836ca5a90>>
+2021-07-09 14:41:04,795 INFO    MainThread:250031 [wandb_run.py:_config_callback():872] config_cb test_log 12345 None
+2021-07-09 15:03:23,473 INFO    MainThread:250031 [wandb_run.py:_tensorboard_callback():943] tensorboard callback: /home/cahya/Work/flax-community/gpt2-medium-indonesian, None