Training in progress, step 2800, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c53cd6b7a9aaec48a0490b1f83024497fb7a940d74235cfb8f5dc2c2a530336
+oid sha256:5d8105e08b44734413954313ee3f59582f2323ad431f91a8d42d31e834f41c4c
 size 18516456

last-checkpoint/global_step2799/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed61ae05c7e207208a57e99e9c7b23ccf0adef6c711c3daf1878f7f7baeb82d4
+size 27700976

last-checkpoint/global_step2799/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d77f154dbfad53dd9c46f2c8758ac2dc27a1b017f9406698b5b26e99636df7d
+size 27700976

last-checkpoint/global_step2799/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f77ae3efa9f4d34bc4fddefb730de1851905c8f19a59dd89433a3198a01db13
+size 27700976

last-checkpoint/global_step2799/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de1f156dd1c74f4abed59f4cb92efe3d6c7bb3c87f2005b1a6d359c059e54d10
+size 27700976

last-checkpoint/global_step2799/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c0be6cc674bb639b4ec7baf63317da27aa182e28279625298e4fb57664d53a
+size 411571

last-checkpoint/global_step2799/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69dd93b13a7c83a007ed01f8158aa0d3c23abff7686d47b321a7585318ae6439
+size 411507

last-checkpoint/global_step2799/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9abca242f770cca89ea6a43c472921b374458558880f50808f2e470bff246a17
+size 411507

last-checkpoint/global_step2799/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f52198786ecf09c894a82afe94ca79b2505b075200315fa9dc59e7cb879e72db
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2749
+global_step2799

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49db5a9fd0c84d580c671e52905ebeffc155b36537e76ff966d2e82906708999
+oid sha256:ff8dba2341c0517760edfde50521977f02a5bd982ffd3bc03de6109439c4f478
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8ca224562d8d97aaa131b3516288bb99f68d7dcf62170494326662bda0bb206
+oid sha256:d2bf831df9fbade9ac2a8db79798bc2a7b1afb85a78a6e463ec7a7db4acc0f8e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56e86a11c89dba78d60e1b2a1855a651b90a5a22ef131ce65d26af83668c154e
+oid sha256:8602ff0a0fa366d46b61c0ef2b23ce468387898cf2bc1027e5450de73ddf647f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3dd63019c923e9692431619aced46b91aaf3fd22e1c22ec0a64347f2fe635a0e
+oid sha256:4bb51d675cf23603b1b765cd645f53d6b66ddb104d56d48674e9c798e086f696
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eac20f941d720434897b4bd1a2af1a5643e9ab3c6b5d2f9906a1dfc05b7c03aa
+oid sha256:61a7c605cf80a46d2e0c661d5469c16671b681f268e3ecd5d1d64188653910db
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.4161730620654134,
+  "epoch": 1.4419263456090652,
   "eval_steps": 50,
-  "global_step": 2750,
+  "global_step": 2800,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4902,11 +4902,100 @@
       "eval_steps_per_second": 0.929,
       "num_input_tokens_seen": 32158144,
       "step": 2750
+    },
+    {
+      "epoch": 1.4187483904197786,
+      "grad_norm": 6.584628814510667,
+      "learning_rate": 9.520584561109864e-06,
+      "loss": 0.3333,
+      "num_input_tokens_seen": 32216656,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4213237187741437,
+      "grad_norm": 7.509676086247465,
+      "learning_rate": 9.378330907169386e-06,
+      "loss": 0.2993,
+      "num_input_tokens_seen": 32275168,
+      "step": 2760
+    },
+    {
+      "epoch": 1.423899047128509,
+      "grad_norm": 5.1775193353141535,
+      "learning_rate": 9.237037961694223e-06,
+      "loss": 0.2683,
+      "num_input_tokens_seen": 32333664,
+      "step": 2765
+    },
+    {
+      "epoch": 1.4264743754828741,
+      "grad_norm": 7.856433365965151,
+      "learning_rate": 9.096709066283354e-06,
+      "loss": 0.3145,
+      "num_input_tokens_seen": 32392088,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4290497038372392,
+      "grad_norm": 8.252870521534577,
+      "learning_rate": 8.957347539735872e-06,
+      "loss": 0.3092,
+      "num_input_tokens_seen": 32450584,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4316250321916044,
+      "grad_norm": 9.74883489294415,
+      "learning_rate": 8.818956677972406e-06,
+      "loss": 0.2993,
+      "num_input_tokens_seen": 32509096,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4342003605459697,
+      "grad_norm": 4.008158818829899,
+      "learning_rate": 8.681539753957269e-06,
+      "loss": 0.326,
+      "num_input_tokens_seen": 32567560,
+      "step": 2785
+    },
+    {
+      "epoch": 1.436775688900335,
+      "grad_norm": 3.4229494980881174,
+      "learning_rate": 8.545100017620988e-06,
+      "loss": 0.2494,
+      "num_input_tokens_seen": 32626056,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4393510172547,
+      "grad_norm": 4.425295787830864,
+      "learning_rate": 8.409640695783443e-06,
+      "loss": 0.2691,
+      "num_input_tokens_seen": 32684520,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4419263456090652,
+      "grad_norm": 5.132559476583136,
+      "learning_rate": 8.275164992077556e-06,
+      "loss": 0.2939,
+      "num_input_tokens_seen": 32743032,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4419263456090652,
+      "eval_loss": 0.791334331035614,
+      "eval_runtime": 16.1142,
+      "eval_samples_per_second": 3.723,
+      "eval_steps_per_second": 0.931,
+      "num_input_tokens_seen": 32743032,
+      "step": 2800
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 32158144,
+  "num_input_tokens_seen": 32743032,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4921,7 +5010,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1805832461746176.0,
+  "total_flos": 1838675721715712.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null