Training in progress, step 1650, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f11513d52bcbabbebdae5c22382754c386d07651a25539e1cdcc0f4b4982a0f7
+oid sha256:919717a8f83cc7597c4b3c1c6f8d6f3d9da7bb13e3737cb159de4e4e604b9ef9
 size 18516456

last-checkpoint/global_step1650/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc3d2bcd74c2289546af424c3ccef41a3590dee4d98692dc7035cb770ec6440c
+size 27700976

last-checkpoint/global_step1650/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc6c1583f6d8a8f5ea608d477928945261e94155b200c9d5296d88728ae362e7
+size 27700976

last-checkpoint/global_step1650/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32b4ffd70ff27d3f3556fb95a47f832ac49820396d6cf7a0e5be6d97cd0bd157
+size 27700976

last-checkpoint/global_step1650/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc9303ec814fb09b35b1d5a95b7b6bf4117a5288d03c0b091aae039b16f327e9
+size 27700976

last-checkpoint/global_step1650/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86a71feee743879831ed6c178773eb14d5a9a8462d065c3a323d963c0abaf6d3
+size 411571

last-checkpoint/global_step1650/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:420d090db4c38cc554ab73a50666f44378418f6870378edf1f5744cc992f4f60
+size 411507

last-checkpoint/global_step1650/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ac1e6c793af99eadc9a824e76c6395926948aa5ed3496c3fcfaac1a0b73cca5
+size 411507

last-checkpoint/global_step1650/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a910d523c55d0b02759778eba3d3644d4af2bd51a9d3fcecee59a7b97a5399
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step1600
+global_step1650

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9279ed4b01716237e789d2631c1f29bc5d43c5633c014d4401de21b672c1b355
+oid sha256:a90384755f5b036b42b1a465b39dbf24a925a02c04294f9d684bc1de7f4db1e5
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca1990d68e57c70df5c56d395dd3f3befbe07b380521f4144677c20f6fe2a3eb
+oid sha256:7621b41e55056218f97d5b32ae116de3304a677b9f27b6a62170d83a2bbff176
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0790066885525e1b9a9390a40ae27abd57abb47f031abface27890732f9e684
+oid sha256:997e9debadfd125b5c8b66ee6dd79ced3d40d353ff9250475f3814fd950012a6
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1325a2034fe48ebad4f00ac8a2b32ab5c4c43c2497712169a8e3b1112363d916
+oid sha256:0f0256032419959580948d742425f66782bc8eb029126a091669a42c6ee0eba4
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2487a6c511ed8055eb0842d87966b09ae8b62c1b4514727282ca413d6e9c4e2
+oid sha256:a67704d17501733446a3916f5d09739e8293f473d2f1a5f63158f87200fdaa72
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 0.8241050733968581,
+  "epoch": 0.8498583569405099,
   "eval_steps": 50,
-  "global_step": 1600,
+  "global_step": 1650,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2855,11 +2855,100 @@
       "eval_steps_per_second": 0.936,
       "num_input_tokens_seen": 18714072,
       "step": 1600
+    },
+    {
+      "epoch": 0.8266804017512233,
+      "grad_norm": 3.4115030121534953,
+      "learning_rate": 5.8709014638958404e-05,
+      "loss": 0.6095,
+      "num_input_tokens_seen": 18772552,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8292557301055885,
+      "grad_norm": 2.8584050529867895,
+      "learning_rate": 5.846947222863123e-05,
+      "loss": 0.5896,
+      "num_input_tokens_seen": 18830992,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8318310584599536,
+      "grad_norm": 3.083134826868609,
+      "learning_rate": 5.8229729514036705e-05,
+      "loss": 0.545,
+      "num_input_tokens_seen": 18889480,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8344063868143188,
+      "grad_norm": 3.5650772646006703,
+      "learning_rate": 5.7989792165125356e-05,
+      "loss": 0.6021,
+      "num_input_tokens_seen": 18947936,
+      "step": 1620
+    },
+    {
+      "epoch": 0.836981715168684,
+      "grad_norm": 3.1787537764025737,
+      "learning_rate": 5.774966585645092e-05,
+      "loss": 0.5741,
+      "num_input_tokens_seen": 19006432,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8395570435230492,
+      "grad_norm": 4.505205596087594,
+      "learning_rate": 5.7509356267035975e-05,
+      "loss": 0.5796,
+      "num_input_tokens_seen": 19064920,
+      "step": 1630
+    },
+    {
+      "epoch": 0.8421323718774144,
+      "grad_norm": 3.854433226263906,
+      "learning_rate": 5.726886908023776e-05,
+      "loss": 0.5088,
+      "num_input_tokens_seen": 19123376,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8447077002317795,
+      "grad_norm": 3.5910960304247643,
+      "learning_rate": 5.702820998361373e-05,
+      "loss": 0.5431,
+      "num_input_tokens_seen": 19181864,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8472830285861447,
+      "grad_norm": 4.55639282269759,
+      "learning_rate": 5.6787384668786994e-05,
+      "loss": 0.5849,
+      "num_input_tokens_seen": 19240352,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8498583569405099,
+      "grad_norm": 4.031478721616991,
+      "learning_rate": 5.654639883131178e-05,
+      "loss": 0.5668,
+      "num_input_tokens_seen": 19298848,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8498583569405099,
+      "eval_loss": 0.6634677648544312,
+      "eval_runtime": 16.0267,
+      "eval_samples_per_second": 3.744,
+      "eval_steps_per_second": 0.936,
+      "num_input_tokens_seen": 19298848,
+      "step": 1650
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 18714072,
+  "num_input_tokens_seen": 19298848,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2874,7 +2963,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1050836028358656.0,
+  "total_flos": 1083674466975744.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null