Training in progress, step 2750, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87c5131642e0134183eead6ee8a652dcec18d08a73880b46e923d3ba034a1f2c
+oid sha256:5c53cd6b7a9aaec48a0490b1f83024497fb7a940d74235cfb8f5dc2c2a530336
 size 18516456

last-checkpoint/global_step2749/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4900d28228b39633c0fc7a89a0d6d17045a472ec14c243c08b3a8a18e265bce0
+size 27700976

last-checkpoint/global_step2749/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc4258fec3a0b749bf4ed1e7249624eefd5a2404f4b71ba2609b9e37f8e36c62
+size 27700976

last-checkpoint/global_step2749/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8537ae171e8a0fef8885e434157bed6132b70a5054ff0b05a542d05e118ac6ee
+size 27700976

last-checkpoint/global_step2749/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:354d3767be78a341d093d69b42dd0564e20a7d0775a2c3315e974d23e2fb4416
+size 27700976

last-checkpoint/global_step2749/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f371e9ce094939d3861a5773cf2cd0dc35662ff54accc1cc07f3b940d7364101
+size 411571

last-checkpoint/global_step2749/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e34a0091e2f2d1677ed1efb3fb25f547edd42c07fd556d39731391c1dacd129
+size 411507

last-checkpoint/global_step2749/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abcaf0f06b18f96340c928d314b3bb483810ab95835fc08c188569390f0a71e9
+size 411507

last-checkpoint/global_step2749/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b70a0800e2d1ff59c82590c16976a86bc1115aadc7292b2d19b08fa9a1c77df
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2699
+global_step2749

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d8d3c7739f9787ea797b86ff1b3a51f9e68197835ba3178915a8a77558f67fc
+oid sha256:49db5a9fd0c84d580c671e52905ebeffc155b36537e76ff966d2e82906708999
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a22a57799bc43e59db67d9a787ed73040020c5f35990602033f4dab1318787d7
+oid sha256:c8ca224562d8d97aaa131b3516288bb99f68d7dcf62170494326662bda0bb206
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29a624b936b77a04d6bfb6940acdd65a710bf39452e419e7ddb5c40fb2261072
+oid sha256:56e86a11c89dba78d60e1b2a1855a651b90a5a22ef131ce65d26af83668c154e
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a79306817d4440cd621149537e8cf216b60f847fc6f9531a6147426aa02bb07
+oid sha256:3dd63019c923e9692431619aced46b91aaf3fd22e1c22ec0a64347f2fe635a0e
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e5dcca4048a125fff8fd284657b0498882f3efcb97d36e331842fc3d6d7b6e6
+oid sha256:eac20f941d720434897b4bd1a2af1a5643e9ab3c6b5d2f9906a1dfc05b7c03aa
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.3904197785217616,
+  "epoch": 1.4161730620654134,
   "eval_steps": 50,
-  "global_step": 2700,
+  "global_step": 2750,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4813,11 +4813,100 @@
       "eval_steps_per_second": 0.928,
       "num_input_tokens_seen": 31573240,
       "step": 2700
+    },
+    {
+      "epoch": 1.3929951068761266,
+      "grad_norm": 4.579483859059419,
+      "learning_rate": 1.0995208772202897e-05,
+      "loss": 0.2798,
+      "num_input_tokens_seen": 31631688,
+      "step": 2705
+    },
+    {
+      "epoch": 1.3955704352304918,
+      "grad_norm": 6.098482033036635,
+      "learning_rate": 1.0843536780343865e-05,
+      "loss": 0.289,
+      "num_input_tokens_seen": 31690200,
+      "step": 2710
+    },
+    {
+      "epoch": 1.398145763584857,
+      "grad_norm": 9.834029857293697,
+      "learning_rate": 1.069279084461513e-05,
+      "loss": 0.2844,
+      "num_input_tokens_seen": 31748664,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4007210919392223,
+      "grad_norm": 9.387518267357049,
+      "learning_rate": 1.0542974530180327e-05,
+      "loss": 0.3254,
+      "num_input_tokens_seen": 31807176,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4032964202935874,
+      "grad_norm": 5.648695214602192,
+      "learning_rate": 1.0394091380217352e-05,
+      "loss": 0.3683,
+      "num_input_tokens_seen": 31865696,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4058717486479526,
+      "grad_norm": 5.202858729177478,
+      "learning_rate": 1.0246144915834683e-05,
+      "loss": 0.2968,
+      "num_input_tokens_seen": 31924200,
+      "step": 2730
+    },
+    {
+      "epoch": 1.4084470770023179,
+      "grad_norm": 4.808429946385537,
+      "learning_rate": 1.0099138635988026e-05,
+      "loss": 0.2943,
+      "num_input_tokens_seen": 31982712,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4110224053566829,
+      "grad_norm": 5.094039780174813,
+      "learning_rate": 9.953076017397578e-06,
+      "loss": 0.3037,
+      "num_input_tokens_seen": 32041176,
+      "step": 2740
+    },
+    {
+      "epoch": 1.4135977337110481,
+      "grad_norm": 5.807237736394797,
+      "learning_rate": 9.807960514465792e-06,
+      "loss": 0.3019,
+      "num_input_tokens_seen": 32099656,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4161730620654134,
+      "grad_norm": 6.27488451409393,
+      "learning_rate": 9.663795559195733e-06,
+      "loss": 0.164,
+      "num_input_tokens_seen": 32158144,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4161730620654134,
+      "eval_loss": 0.7807286381721497,
+      "eval_runtime": 16.139,
+      "eval_samples_per_second": 3.718,
+      "eval_steps_per_second": 0.929,
+      "num_input_tokens_seen": 32158144,
+      "step": 2750
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 31573240,
+  "num_input_tokens_seen": 32158144,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4832,7 +4921,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1772988198813696.0,
+  "total_flos": 1805832461746176.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null