Training in progress, step 18660

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ee7bcf91faa799f3b50658aff33fa5ee549302575087b3dff52ae4b72610fef
+oid sha256:b6b239758e5710fe78fd10d067ef696b9ecf7a57962dcbfc99ea16b5f0f15474
 size 13982248

runs/Apr12_11-58-09_f044666cc877/events.out.tfevents.1712923163.f044666cc877.110.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:771208a6e89f60fbedab34929f3248a1b91352640ffad51f4d5bbdbe3d80ef43
-size 201932
+oid sha256:93043374edb9268fcec2f6136bf8524e07171b03523a495ff064d45879c09846
+size 202577

wandb/debug-internal.log

@@ -13922,3 +13922,49 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 17:26:49,433 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 17:26:49,434 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 17:26:49,893 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:26:51,894 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:26:53,100 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:26:53,101 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:57,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:57,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:57,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:58,625 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:03,626 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:08,627 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:10,821 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:27:10,822 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:27:10,822 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:27:10,823 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:27:10,902 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:27:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:27:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:27:12,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:27:13,904 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:27:14,564 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:19,565 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:22,999 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:27:25,000 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:27,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:27:27,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:27:27,515 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:27:30,589 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:30,835 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:27:30,836 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:27:30,836 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:27:30,838 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:27:30,911 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:27:33,912 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:27:35,615 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:40,616 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:42,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:27:42,512 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:27:42,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:27:46,598 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:51,599 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:52,350 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:27:52,351 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:27:52,351 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:27:52,351 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:27:52,919 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:27:53,000 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:27:53,919 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log

wandb/run-20240412_115951-o0z5y86z/files/output.log

@@ -1860,4 +1860,12 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.5333, "train/grad_norm": 31.87932014465332, "train/learning_rate": 6.397368838268497e-07, "train/epoch": 0.26, "train/global_step": 18600, "_timestamp": 1712942809.4317503, "_runtime": 19617.84078025818, "_step": 929}
+{"train/loss": 3.4893, "train/grad_norm": 19.461843490600586, "train/learning_rate": 5.862880896264688e-07, "train/epoch": 0.26, "train/global_step": 18660, "_timestamp": 1712942872.3497894, "_runtime": 19680.75881934166, "_step": 932}

wandb/run-20240412_115951-o0z5y86z/logs/debug-internal.log

@@ -13922,3 +13922,49 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-12 17:26:49,433 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
 2024-04-12 17:26:49,434 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-12 17:26:49,893 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:26:51,894 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:26:53,100 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:26:53,101 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:26:57,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:26:57,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:26:57,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:26:58,625 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:03,626 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:08,627 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:10,821 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:27:10,822 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:27:10,822 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:27:10,823 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:27:10,902 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:27:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:27:12,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:27:12,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:27:13,904 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:27:14,564 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:19,565 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:22,999 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:27:25,000 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:27,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:27:27,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:27:27,515 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:27:30,589 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:30,835 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:27:30,836 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:27:30,836 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:27:30,838 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:27:30,911 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:27:33,912 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log
+2024-04-12 17:27:35,615 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:40,616 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:42,511 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-12 17:27:42,512 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-12 17:27:42,512 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: stop_status
+2024-04-12 17:27:46,598 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:51,599 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: status_report
+2024-04-12 17:27:52,350 DEBUG   HandlerThread:151 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-12 17:27:52,351 DEBUG   SenderThread:151 [sender.py:send():379] send: history
+2024-04-12 17:27:52,351 DEBUG   SenderThread:151 [sender.py:send_request():406] send_request: summary_record
+2024-04-12 17:27:52,351 INFO    SenderThread:151 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-12 17:27:52,919 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/wandb-summary.json
+2024-04-12 17:27:53,000 DEBUG   SenderThread:151 [sender.py:send():379] send: stats
+2024-04-12 17:27:53,919 INFO    Thread-12 :151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240412_115951-o0z5y86z/files/output.log

wandb/run-20240412_115951-o0z5y86z/run-o0z5y86z.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14f25204bb53d2ea3d718c32ff6e34e885fdb002a5de4ef2c0f53b3686676d7a
-size 1307440
+oid sha256:9e27f1e08ad3e0e129ab6ddfbf4c7d7954d6bbdf0ad21c6762e570aec71697b3
+size 1314361