Saving weights and logs of step 50

events.out.tfevents.1625840127.t1v-n-528d9406-w-0.245719.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64a1f53cff36f1ceef50b134d2b7ac7ac032f23128f8144fad2f113039c49027
-size 600
+oid sha256:29127ff00a5abb5eced7951908274665a9e0884b0e11a56e7587706a835ed86b
+size 740

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42da66777ec7c8efce01daf35cfa7c98bfa70b0dc0f0bff82a13bead069b2f75
+oid sha256:7d2e5440f36a01e368586fdd77a1ab6a8745b8d0b1a7dd09510558e9905e4c3f
 size 1419302302

wandb/run-20210709_141445-2k8cnty2/files/output.log

@@ -474,4 +474,14 @@ Training...:  77%|████████████████████
 Evaluating...:  20%|███████████████████████████████████████                                                                                                                                                            | 1/5 [00:05<00:22,  5.72s/it]
 
 
+tcmalloc: large alloc 2426904576 bytes == 0x56396efea000 @  0x7faff8cbc680 0x7faff8cdcbdd 0x7faca7dcf20d 0x7faca7ddd340 0x7faca7ddce87 0x7faca7ddce87 0x7faca7ddce87 0x7faca7dd8bd3 0x7faca7dd91fe 0x56364a10452a 0x56364a0cb254 0x56364a11c9c9 0x56364a11d85f 0x56364a0cacc7 0x56364a11d768 0x56364a0cae14 0x56364a11d768 0x56364a0cae14 0x56364a11d066 0x56364a1284c6 0x56364a0cacc7 0x56364a11d132 0x56364a11d7fb 0x56364a0cae14 0x56364a11c9c9 0x56364a17c383 0x56364a17c8d2 0x56364a18c982 0x56364a18e429 0x56364a18e60f 0x56364a18ea09
+Model weights saved in /home/cahya/Work/flax-community/gpt2-medium-indonesian/flax_model.msgpack
+Model pushed to the hub in this commit: https://huggingface.co/flax-community/gpt2-medium-indonesian/commit/c5559541cc37da7694d050a4d8b457ea760ce498
+
+
+Training...:  96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌       | 50/52 [06:30<00:03,  1.88s/it]
+
+Evaluating...:   0%|                                                                                                                                                                                                           | 0/5 [00:00<?, ?it/s]
+
+
 tcmalloc: large alloc 2426904576 bytes == 0x56396efea000 @  0x7faff8cbc680 0x7faff8cdcbdd 0x7faca7dcf20d 0x7faca7ddd340 0x7faca7ddce87 0x7faca7ddce87 0x7faca7ddce87 0x7faca7dd8bd3 0x7faca7dd91fe 0x56364a10452a 0x56364a0cb254 0x56364a11c9c9 0x56364a11d85f 0x56364a0cacc7 0x56364a11d768 0x56364a0cae14 0x56364a11d768 0x56364a0cae14 0x56364a11d066 0x56364a1284c6 0x56364a0cacc7 0x56364a11d132 0x56364a11d7fb 0x56364a0cae14 0x56364a11c9c9 0x56364a17c383 0x56364a17c8d2 0x56364a18c982 0x56364a18e429 0x56364a18e60f 0x56364a18ea09

wandb/run-20210709_141445-2k8cnty2/files/wandb-summary.json

@@ -1 +1 @@
-{"global_step": 30, "_timestamp": 1625840355.097392, "eval_loss": 8.853381156921387, "eval_perplexity": 6998.01025390625, "_step": 2}
+{"global_step": 40, "_timestamp": 1625840437.50007, "eval_loss": 8.340357780456543, "eval_perplexity": 4189.58837890625, "_step": 3}

wandb/run-20210709_141445-2k8cnty2/logs/debug-internal.log

@@ -168,3 +168,33 @@
 2021-07-09 14:20:53,491 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-09 14:20:53,491 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
 2021-07-09 14:20:57,442 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:21:08,625 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:21:08,625 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:21:16,486 DEBUG   SenderThread:246776 [sender.py:send():179] send: stats
+2021-07-09 14:21:23,754 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:21:23,754 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:21:38,887 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:21:38,888 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:21:46,566 DEBUG   SenderThread:246776 [sender.py:send():179] send: stats
+2021-07-09 14:21:54,019 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:21:54,020 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:21:56,465 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:21:58,465 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:00,466 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:02,467 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:08,113 DEBUG   SenderThread:246776 [sender.py:send():179] send: history
+2021-07-09 14:22:08,114 DEBUG   SenderThread:246776 [sender.py:send():179] send: summary
+2021-07-09 14:22:08,114 INFO    SenderThread:246776 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-09 14:22:08,469 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:08,470 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/events.out.tfevents.1625840127.t1v-n-528d9406-w-0.245719.3.v2
+2021-07-09 14:22:08,470 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/wandb-summary.json
+2021-07-09 14:22:09,013 INFO    Thread-18 :246776 [upload_job.py:push():137] Uploaded file /tmp/tmp4f0a6i2mwandb/otuqhtyr-events.out.tfevents.1625840127.t1v-n-528d9406-w-0.245719.3.v2
+2021-07-09 14:22:09,151 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:22:09,151 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:22:09,470 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:10,470 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:16,473 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log
+2021-07-09 14:22:16,645 DEBUG   SenderThread:246776 [sender.py:send():179] send: stats
+2021-07-09 14:22:24,304 DEBUG   HandlerThread:246776 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-09 14:22:24,304 DEBUG   SenderThread:246776 [sender.py:send_request():193] send_request: stop_status
+2021-07-09 14:22:28,478 INFO    Thread-8  :246776 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/cahya/Work/flax-community/gpt2-medium-indonesian/wandb/run-20210709_141445-2k8cnty2/files/output.log