Training in progress, step 100

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e72960fe1f5461766c235a01313cd773da95227563f32936ac3a015ff0459139
+oid sha256:04d2ee0f7546c8480db5f6b7431a5ecf9eddea3854bdd7845dc94b84bfe03f1e
 size 12803224

runs/Apr14_07-30-34_f694866fb244/events.out.tfevents.1713079834.f694866fb244.141.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c48991b5037fa3580978f770e79225d38022d877d613bc121c590e9201fa2d3
-size 6108
+oid sha256:b692c03e71affb56d97fe859c81f664356993b9cf7317f21fbc4dc7d2208675b
+size 6315

wandb/debug-internal.log

@@ -127,3 +127,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-14 07:32:59,582 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: summary_record
 2024-04-14 07:32:59,582 INFO    SenderThread:183 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-14 07:33:00,154 INFO    Thread-12 :183 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240414_073117-xuqlocdo/files/wandb-summary.json
+2024-04-14 07:33:02,155 INFO    Thread-12 :183 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240414_073117-xuqlocdo/files/output.log
+2024-04-14 07:33:03,319 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:04,298 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-14 07:33:04,299 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: stop_status
+2024-04-14 07:33:04,301 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-14 07:33:08,369 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:13,370 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:18,240 DEBUG   SenderThread:183 [sender.py:send():379] send: stats
+2024-04-14 07:33:19,241 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:19,298 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-14 07:33:19,299 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: stop_status
+2024-04-14 07:33:19,339 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-14 07:33:22,849 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-14 07:33:22,850 DEBUG   SenderThread:183 [sender.py:send():379] send: history
+2024-04-14 07:33:22,851 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: summary_record
+2024-04-14 07:33:22,852 INFO    SenderThread:183 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-14 07:33:23,163 INFO    Thread-12 :183 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240414_073117-xuqlocdo/files/wandb-summary.json

wandb/run-20240414_073117-xuqlocdo/files/output.log

@@ -4,4 +4,6 @@
   warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240414_073117-xuqlocdo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 4.0846, "train/grad_norm": 109.34711456298828, "train/learning_rate": 1.2e-05, "train/epoch": 0.0, "train/global_step": 80, "_timestamp": 1713079979.5805168, "_runtime": 102.14129376411438, "_step": 3}
+{"train/loss": 3.594, "train/grad_norm": 75.66071319580078, "train/learning_rate": 1.5333333333333334e-05, "train/epoch": 0.0, "train/global_step": 100, "_timestamp": 1713080002.8489428, "_runtime": 125.40971970558167, "_step": 4}

wandb/run-20240414_073117-xuqlocdo/logs/debug-internal.log

@@ -127,3 +127,20 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-14 07:32:59,582 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: summary_record
 2024-04-14 07:32:59,582 INFO    SenderThread:183 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-14 07:33:00,154 INFO    Thread-12 :183 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240414_073117-xuqlocdo/files/wandb-summary.json
+2024-04-14 07:33:02,155 INFO    Thread-12 :183 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240414_073117-xuqlocdo/files/output.log
+2024-04-14 07:33:03,319 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:04,298 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-14 07:33:04,299 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: stop_status
+2024-04-14 07:33:04,301 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-14 07:33:08,369 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:13,370 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:18,240 DEBUG   SenderThread:183 [sender.py:send():379] send: stats
+2024-04-14 07:33:19,241 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: status_report
+2024-04-14 07:33:19,298 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-14 07:33:19,299 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: stop_status
+2024-04-14 07:33:19,339 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-14 07:33:22,849 DEBUG   HandlerThread:183 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-14 07:33:22,850 DEBUG   SenderThread:183 [sender.py:send():379] send: history
+2024-04-14 07:33:22,851 DEBUG   SenderThread:183 [sender.py:send_request():406] send_request: summary_record
+2024-04-14 07:33:22,852 INFO    SenderThread:183 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-14 07:33:23,163 INFO    Thread-12 :183 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240414_073117-xuqlocdo/files/wandb-summary.json