Training in progress, step 3250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf91cb70edf4762f7eda2712d2bbbe2e8db267086af287aae6ab31428da6d543
+oid sha256:bcd2ef8b4ec22a6a44f487539bb38bf8a3cfbf6e4cda8dbbad18255c7b196a6b
 size 18516456

last-checkpoint/global_step3249/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:239c5f72e7fca7b1304d8a4c31d62020adbd183c86ac72f269fbd1590d050199
+size 27700976

last-checkpoint/global_step3249/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:707499cb541bde5ccd0e2d3dcaebca558ae0a1e6e96f9fde4fb8fab003e298fd
+size 27700976

last-checkpoint/global_step3249/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b0ddf2cfb1aa13d296518d2f9bf7111853dd8ea3e978909b2b973f065667360
+size 27700976

last-checkpoint/global_step3249/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d4f2bcf01b5213ab6bc2bb0e2ed279c82a77dcc5ce2197e6e4e891a361a66fc
+size 27700976

last-checkpoint/global_step3249/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:802974d7bd7e6d8cf5121193e585f69c25cd6ef721c08f9970ec32bf943c6565
+size 411571

last-checkpoint/global_step3249/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3156e0133b88013b5a3b37d156dc87b2a279a71326ba4e26f91af72251348a3a
+size 411507

last-checkpoint/global_step3249/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8747465b7428b359cc78c7bfb11588184857000949ea2c257aac39f8711d174a
+size 411507

last-checkpoint/global_step3249/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f93606d190c084485ab366f86527d0690c2836a92abfc67250c9bdf2a28ef839
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step3199
+global_step3249

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbe41a7cf0c0baf2cbd9c6a8c2d572d7807492b0cef50528f3655343ea1b64d5
+oid sha256:9d8a584abdea9bed1d2dc22d8c9034ba07ae85ad6668d6cd55747af816dcecc1
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a52ad24d6a9f3bd519fc26476a8d09236301af97836545a4d2ac6bddc629489e
+oid sha256:7862a59bb6480df16b96e840172772c30d9d3037819d18063df33360e83b04a2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67e13ea1a63841414e59ea8fa8ddcdbabf6d8e38c6583e14a24c83f676a1e986
+oid sha256:3b509c445a6a6ed6d0215c0e232a66489dc20e39b5202d274561183fd43ef3e5
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c233218300ca098f4464c8e75ae36c65a8a1eaae6f304c77db92627a8b385448
+oid sha256:1cf1a6a26872b9918c1086ace4fcda88a76d9b6eef501300fd67bf0cc8946d81
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4756a584dac90cc9bdfc6529aec3d745541ed8f41536120e43dc084a360376c
+oid sha256:513043e75e41275111fc2a7e55fb4c3a862256ab8b8062676e12ebb4154e419d
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.6479526139582796,
+  "epoch": 1.6737058975019314,
   "eval_steps": 50,
-  "global_step": 3200,
+  "global_step": 3250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5703,11 +5703,100 @@
       "eval_steps_per_second": 0.928,
       "num_input_tokens_seen": 37421416,
       "step": 3200
+    },
+    {
+      "epoch": 1.6505279423126449,
+      "grad_norm": 12.027460444161642,
+      "learning_rate": 8.966060675892951e-07,
+      "loss": 0.2865,
+      "num_input_tokens_seen": 37479848,
+      "step": 3205
+    },
+    {
+      "epoch": 1.65310327066701,
+      "grad_norm": 6.851221931248735,
+      "learning_rate": 8.513450158049108e-07,
+      "loss": 0.3299,
+      "num_input_tokens_seen": 37538312,
+      "step": 3210
+    },
+    {
+      "epoch": 1.6556785990213752,
+      "grad_norm": 6.971651790450948,
+      "learning_rate": 8.072463369597993e-07,
+      "loss": 0.3218,
+      "num_input_tokens_seen": 37596800,
+      "step": 3215
+    },
+    {
+      "epoch": 1.6582539273757404,
+      "grad_norm": 10.994527310957624,
+      "learning_rate": 7.643110739942172e-07,
+      "loss": 0.2593,
+      "num_input_tokens_seen": 37655312,
+      "step": 3220
+    },
+    {
+      "epoch": 1.6608292557301056,
+      "grad_norm": 13.542379224085927,
+      "learning_rate": 7.225402423334693e-07,
+      "loss": 0.3072,
+      "num_input_tokens_seen": 37713800,
+      "step": 3225
+    },
+    {
+      "epoch": 1.663404584084471,
+      "grad_norm": 5.442561929450427,
+      "learning_rate": 6.819348298638839e-07,
+      "loss": 0.2276,
+      "num_input_tokens_seen": 37772280,
+      "step": 3230
+    },
+    {
+      "epoch": 1.665979912438836,
+      "grad_norm": 8.128386248398428,
+      "learning_rate": 6.424957969094536e-07,
+      "loss": 0.2489,
+      "num_input_tokens_seen": 37830800,
+      "step": 3235
+    },
+    {
+      "epoch": 1.6685552407932012,
+      "grad_norm": 3.9766881915113266,
+      "learning_rate": 6.0422407620912e-07,
+      "loss": 0.2552,
+      "num_input_tokens_seen": 37889280,
+      "step": 3240
+    },
+    {
+      "epoch": 1.6711305691475662,
+      "grad_norm": 5.555365927504982,
+      "learning_rate": 5.671205728947305e-07,
+      "loss": 0.226,
+      "num_input_tokens_seen": 37947728,
+      "step": 3245
+    },
+    {
+      "epoch": 1.6737058975019314,
+      "grad_norm": 5.733028191926084,
+      "learning_rate": 5.311861644696048e-07,
+      "loss": 0.2785,
+      "num_input_tokens_seen": 38006200,
+      "step": 3250
+    },
+    {
+      "epoch": 1.6737058975019314,
+      "eval_loss": 0.8640011548995972,
+      "eval_runtime": 16.0965,
+      "eval_samples_per_second": 3.728,
+      "eval_steps_per_second": 0.932,
+      "num_input_tokens_seen": 38006200,
+      "step": 3250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 37421416,
+  "num_input_tokens_seen": 38006200,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5722,7 +5811,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2101393008361472.0,
+  "total_flos": 2134231948460032.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null