Training in progress, step 3050, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0c6af243f589a55fa9a3c1d1d73c19fe4b53c2830aa75e5338a55fa38ab436d
+oid sha256:ddb314047f6c123f2abaf4b7b141070b2299dee4fc65b9c0ec7277494419776e
 size 18516456

last-checkpoint/global_step3049/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f5f01fb31fdf05d1e57447dec005c951bccf5c68312950d9ba68ed2b72acc3f
+size 27700976

last-checkpoint/global_step3049/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57fb41b9d59335b24ba9718ac6c9fd7127282cbbe0d7b4b41a3649591408f50b
+size 27700976

last-checkpoint/global_step3049/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e952f98dd8858eb80ad86f5a924c0ad5ab72100eb02d784dc6a1b68d87f21e06
+size 27700976

last-checkpoint/global_step3049/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5e81e920bc6cb45f20660b91acbcade620837849e36b76142db5dac6cd1f9cb
+size 27700976

last-checkpoint/global_step3049/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8471330bf5a8c63f2b89d46daf4f4aaad340a671bd43e8f5bc9e07681597db2f
+size 411571

last-checkpoint/global_step3049/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76adf22263ee0fa2a57671c2d94a26406cb5e23e4e71ca187da62197af0a3f48
+size 411507

last-checkpoint/global_step3049/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2ece78d03d8ad6a00affb15a09e2fd2c6e6f387952a1aa180bc85011e19774a
+size 411507

last-checkpoint/global_step3049/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dccca4b860945d3665f2eca85ecde9be91b246c79f8a168bc5072e8f5c567d74
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2999
+global_step3049

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f30f7f3c3620ccd30020e1ea4b81d1a56ee511b742c91370577d55399c14412
+oid sha256:381f090b12cbb3fcce976bd2e72d07a7786154c2848cc881d75715648e7c4dc2
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:440cc1f8e138e1a90606722bd350460b4460991a2f3671f46f880f5743522dca
+oid sha256:4321dc1beb6c245d4a16a8fb2bed2ce2a40d89e5a9c611c4572f63a09523846c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d78d5df466f5b34e85649b90825d4b168464f6d49c668313415473184409b799
+oid sha256:5b79020117e301cd96518c9d1e3eb43a609bc85799f14bac63d9c572fc04cb89
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d8be6df9e66c8bba407df289d2aa6b4422668727a1e2419037de032213393a4
+oid sha256:5429623c7323b4a820ea3d76194bc456c2affeeb1f3af8978aec5aee11b2d1ef
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d143caf68fd1aa97e1a4c3e95f828909a9de86a4e283fcdef513f7e5c5c180e5
+oid sha256:65132092aa084390575a45bea99d0dcabb8005a8fae760edf38c6251571f2afa
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.5449394797836724,
+  "epoch": 1.5706927633273242,
   "eval_steps": 50,
-  "global_step": 3000,
+  "global_step": 3050,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5347,11 +5347,100 @@
       "eval_steps_per_second": 0.925,
       "num_input_tokens_seen": 35082056,
       "step": 3000
+    },
+    {
+      "epoch": 1.5475148081380374,
+      "grad_norm": 5.643348291984009,
+      "learning_rate": 3.644856725057405e-06,
+      "loss": 0.2157,
+      "num_input_tokens_seen": 35140568,
+      "step": 3005
+    },
+    {
+      "epoch": 1.5500901364924027,
+      "grad_norm": 6.225693907549098,
+      "learning_rate": 3.554268172739661e-06,
+      "loss": 0.2233,
+      "num_input_tokens_seen": 35199064,
+      "step": 3010
+    },
+    {
+      "epoch": 1.552665464846768,
+      "grad_norm": 5.080945994557626,
+      "learning_rate": 3.4647780688298826e-06,
+      "loss": 0.2951,
+      "num_input_tokens_seen": 35257576,
+      "step": 3015
+    },
+    {
+      "epoch": 1.5552407932011332,
+      "grad_norm": 5.263879934995459,
+      "learning_rate": 3.376388529782215e-06,
+      "loss": 0.2274,
+      "num_input_tokens_seen": 35316064,
+      "step": 3020
+    },
+    {
+      "epoch": 1.5578161215554984,
+      "grad_norm": 5.655349471422181,
+      "learning_rate": 3.2891016460222967e-06,
+      "loss": 0.2479,
+      "num_input_tokens_seen": 35374504,
+      "step": 3025
+    },
+    {
+      "epoch": 1.5603914499098637,
+      "grad_norm": 7.871895425892081,
+      "learning_rate": 3.2029194818977983e-06,
+      "loss": 0.292,
+      "num_input_tokens_seen": 35432984,
+      "step": 3030
+    },
+    {
+      "epoch": 1.5629667782642287,
+      "grad_norm": 6.441418084723481,
+      "learning_rate": 3.117844075629617e-06,
+      "loss": 0.241,
+      "num_input_tokens_seen": 35491488,
+      "step": 3035
+    },
+    {
+      "epoch": 1.5655421066185937,
+      "grad_norm": 5.268339109046189,
+      "learning_rate": 3.033877439263666e-06,
+      "loss": 0.228,
+      "num_input_tokens_seen": 35549984,
+      "step": 3040
+    },
+    {
+      "epoch": 1.568117434972959,
+      "grad_norm": 7.110464304213341,
+      "learning_rate": 2.951021558623274e-06,
+      "loss": 0.2485,
+      "num_input_tokens_seen": 35608488,
+      "step": 3045
+    },
+    {
+      "epoch": 1.5706927633273242,
+      "grad_norm": 12.567694093056492,
+      "learning_rate": 2.869278393262226e-06,
+      "loss": 0.2851,
+      "num_input_tokens_seen": 35666976,
+      "step": 3050
+    },
+    {
+      "epoch": 1.5706927633273242,
+      "eval_loss": 0.8473746180534363,
+      "eval_runtime": 16.0314,
+      "eval_samples_per_second": 3.743,
+      "eval_steps_per_second": 0.936,
+      "num_input_tokens_seen": 35666976,
+      "step": 3050
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 35082056,
+  "num_input_tokens_seen": 35666976,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5366,7 +5455,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1970024459010048.0,
+  "total_flos": 2002870983983104.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null