Training in progress, step 14220

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 bloomz-1b1-vn-chat/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 tmp_trainer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+wandb/run-20240412_115951-o0z5y86z/run-o0z5y86z.wandb filter=lfs diff=lfs merge=lfs -text

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c223a46d77a0541dcf420db07f84a2df91cb6efadc0cbaf5a4f31fc8b2d4bda2
+oid sha256:acf95b3c44e4a266d8d6218ca89e522dcbf1f6fb4cab97c1ace96126a841116e
 size 13982248

runs/Apr12_11-58-09_f044666cc877/events.out.tfevents.1712923163.f044666cc877.110.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8deed49f9be29bb061508eaf1c01033fb07729d17a61a5f003782e262b0edd0
-size 155068
+oid sha256:42252d3c1fcd89afd5d2fb200fbb77467d38988cc68a53d05f005db2a3218eaf
+size 155279

wandb/debug-internal.log

@@ -10636,3 +10636,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 16:09:10,333 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 16:09:10,334 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 16:09:11,061 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 16:09:12,158 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 16:09:12,161 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 16:09:12,161 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 16:09:13,062 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 16:09:13,298 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:18,299 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:22,851 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 16:09:23,852 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:27,158 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 16:09:27,161 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 16:09:27,161 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 16:09:29,260 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:31,544 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 16:09:31,545 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 16:09:31,546 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 16:09:31,546 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 16:09:32,069 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5437, "train/grad_norm": 23.29683494567871, "train/learning_rate": 1.0239940674851941e-05, "train/epoch": 0.2, "train/global_step": 14200, "_timestamp": 1712938150.3315828, "_runtime": 14958.740612745285, "_step": 709}
+{"train/loss": 3.671, "train/grad_norm": 23.486143112182617, "train/learning_rate": 1.0174667483053682e-05, "train/epoch": 0.2, "train/global_step": 14220, "_timestamp": 1712938171.5441086, "_runtime": 14979.953138589859, "_step": 710}

wandb/run-20240412_115951-o0z5y86z/logs/debug-internal.log

@@ -10636,3 +10636,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 16:09:10,333 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 16:09:10,334 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 16:09:11,061 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 16:09:12,158 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 16:09:12,161 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 16:09:12,161 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 16:09:13,062 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 16:09:13,298 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:18,299 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:22,851 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 16:09:23,852 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:27,158 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 16:09:27,161 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 16:09:27,161 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 16:09:29,260 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 16:09:31,544 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 16:09:31,545 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 16:09:31,546 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 16:09:31,546 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 16:09:32,069 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json