Training in progress, step 1250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:39e915e79b012114953a1589b00c8900ef9e4855bec6bd8aa9a11edc484871cf
+oid sha256:7be96f406ec55e2af59cd5b109f6e76e8a66fbf4acb9841a45fb6d6c486af7d2
 size 18516456

last-checkpoint/global_step1250/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26d09c9878e4f364e6befa045ded86fab302872674a59212b06946d48f71a3c3
+size 27700976

last-checkpoint/global_step1250/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886bc6c03f6cad686e8dd5c787093beeb2bc00a1998bf1af23028d58b438c19f
+size 27700976

last-checkpoint/global_step1250/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47214be91922f9512a7d8629e3d1f57cce2325e145959a01509955e36efeeaf6
+size 27700976

last-checkpoint/global_step1250/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beb878cc2717bc78d554c6024454aab9d12e9133b4c67d70fb744bfc48093465
+size 27700976

last-checkpoint/global_step1250/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b63ad4dccc38b42d9e0c6d07c7d8b737dfc65e0309919f86bae8e3dcb0bcafc
+size 411571

last-checkpoint/global_step1250/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce32ab81f64451d7ae4cfc34fc23370ba1931f6e3064a5e10d1c060bd513c1a1
+size 411507

last-checkpoint/global_step1250/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5690617325b666134fb4899a7252c0c721cd060c834a7b7ea14b436a5a1513bb
+size 411507

last-checkpoint/global_step1250/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec5f102391e11f082bf283dfedabe7199e9df698d2f513d3e140cd8748b7627
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1200
+global_step1250

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a209a0c0025f9ce8e2beeba50c1f0828d5c34a2482310fcd0bf5fc24c2c67be2
+oid sha256:15bbef9a4c878a05b4f189e9f77701a153dc9faf093499714094cb36ac0ca030
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a67fb929b8c51f9b1c6ff9f11366e57e55128a1d36df85a9d37a008b49017a75
+oid sha256:7a7122760cfda3f3e13eac4f7e56d09b0fd3beac4bd49b2f0e58da9519469ff9
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b9ef3b0c0978d0b611f4257c939f1c2c6f07e6227bfea6675532d285b0b64a7
+oid sha256:71edf044d4ea96e62ffecab515d42b58ae56cd64abd0092a897468e0e0d7c10f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0183d14c8ed52ee533139532e9bcf7bc34ec297a064845b35741cb501d92675f
+oid sha256:ef5ca30351deca9d83675a183b2186ef346093f56e0d5a6dacdf62ef921f12c2
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16e0cffc6b063574ed312ee2198c86d3dddf2450d0400f042fdc08bd22dcbc7f
+oid sha256:e821d0d4ad1c6c22d1f5fd3b6821d2a85d44ee21d649d544e34dfcb627d9b580
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.7593821287155151,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1200",
-  "epoch": 0.6180788050476436,
+  "best_metric": 0.7407085299491882,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1250",
+  "epoch": 0.6438320885912954,
   "eval_steps": 50,
-  "global_step": 1200,
+  "global_step": 1250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2143,11 +2143,100 @@
       "eval_steps_per_second": 0.93,
       "num_input_tokens_seen": 14035544,
       "step": 1200
+    },
+    {
+      "epoch": 0.6206541334020087,
+      "grad_norm": 1.0526811295206564,
+      "learning_rate": 7.673382966299163e-05,
+      "loss": 0.7871,
+      "num_input_tokens_seen": 14094024,
+      "step": 1205
+    },
+    {
+      "epoch": 0.623229461756374,
+      "grad_norm": 1.832697637344859,
+      "learning_rate": 7.65280325993715e-05,
+      "loss": 0.7594,
+      "num_input_tokens_seen": 14152504,
+      "step": 1210
+    },
+    {
+      "epoch": 0.6258047901107391,
+      "grad_norm": 1.6875031192331054,
+      "learning_rate": 7.63216081438678e-05,
+      "loss": 0.7833,
+      "num_input_tokens_seen": 14210992,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6283801184651043,
+      "grad_norm": 1.867117238207419,
+      "learning_rate": 7.611456117844934e-05,
+      "loss": 0.8445,
+      "num_input_tokens_seen": 14269488,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6309554468194695,
+      "grad_norm": 0.9089614634143406,
+      "learning_rate": 7.59068965998074e-05,
+      "loss": 0.7857,
+      "num_input_tokens_seen": 14327968,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6335307751738347,
+      "grad_norm": 2.3911537408111214,
+      "learning_rate": 7.569861931923989e-05,
+      "loss": 0.8064,
+      "num_input_tokens_seen": 14386448,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6361061035281999,
+      "grad_norm": 1.6500224851295993,
+      "learning_rate": 7.548973426253521e-05,
+      "loss": 0.7117,
+      "num_input_tokens_seen": 14444912,
+      "step": 1235
+    },
+    {
+      "epoch": 0.638681431882565,
+      "grad_norm": 1.508924461189316,
+      "learning_rate": 7.528024636985575e-05,
+      "loss": 0.7449,
+      "num_input_tokens_seen": 14503392,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6412567602369302,
+      "grad_norm": 1.3801142620835953,
+      "learning_rate": 7.507016059562107e-05,
+      "loss": 0.7507,
+      "num_input_tokens_seen": 14561872,
+      "step": 1245
+    },
+    {
+      "epoch": 0.6438320885912954,
+      "grad_norm": 1.2994701535106117,
+      "learning_rate": 7.485948190839077e-05,
+      "loss": 0.7917,
+      "num_input_tokens_seen": 14620336,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6438320885912954,
+      "eval_loss": 0.7407085299491882,
+      "eval_runtime": 16.1168,
+      "eval_samples_per_second": 3.723,
+      "eval_steps_per_second": 0.931,
+      "num_input_tokens_seen": 14620336,
+      "step": 1250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 14035544,
+  "num_input_tokens_seen": 14620336,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2162,7 +2251,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 788117261910016.0,
+  "total_flos": 820953557762048.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null