Training in progress, step 250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbd4c2db163eed1692965995fff6a99c384f9785d5dc16340b44ab7c2b734df0
+oid sha256:3a83521c4411b458bb33d399cce26d96c4156800b9fbac089cf4b485c06b1beb
 size 18516456

last-checkpoint/global_step250/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8490ac3b4e42b8e96c609dfccfb244815e525da5ed46784c5fa307623948c48
+size 27700976

last-checkpoint/global_step250/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f456ad89433bbd2cba3b966643f40f2ae67c4593d529e08627c4c524bfbe644
+size 27700976

last-checkpoint/global_step250/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3dd00ae726181ad2393c9308f51d629c076a4f5bbc657cb4fd97379d597ff10
+size 27700976

last-checkpoint/global_step250/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d03588e23a4a10fa363152bd7550384793c96c83c9aebc8b438d64fe5d94edd
+size 27700976

last-checkpoint/global_step250/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3334fbd60e882da9f52e7a24172b30bf50e01c5da225855026ffe00d39c9afe5
+size 411571

last-checkpoint/global_step250/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9923ee7addf757677f6b94cf75eca3a6705f2bd5f1487bffc8d743a3143b1a7
+size 411507

last-checkpoint/global_step250/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ddf1737f2f4c3240aa49d7dd8dd2499c5ac67895a4b096d3ab8e71c7d3cfbd3
+size 411507

last-checkpoint/global_step250/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2407f0590926452cf77c11720fefdef7f63083250b0f1c1fc29471dff91a9af2
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step200
+global_step250

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c5e18f922d0af74d820247ae97bee506ab412554a58345ddf2558abc94ee3e3
+oid sha256:418a5f105ae834c3075024076916b2a9475918fe034c12d0dd5b6d91f1aba467
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a2dcca6d9741f46592359768ea2212b9321da6408d1fd7d3a80b017bf37f434
+oid sha256:6e07ace389d24bc1307b74f42a1e7b8f0117b0db853e2df64ff3f15cb92916a2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69420ece2c255923c5cbb3c6c9c4a6b9cb38fb57e5d3033c8b7d436a1faf6f13
+oid sha256:da6a990f346d7014dffb28fa2bc7d3b890bd3c53712503fce3656da48d3d6e50
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:66f278b40a1e23b88a657c4e5d03afa8dbbbe14dfeb16f6b4beedaece6cdd0b9
+oid sha256:e95f356ca38179b05993f55daece0223e96fa10b9a1b9ea2102a739211333f63
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdde69e82f45c9368fe627c2082d010eb77a8ceb0b2354f60caa089407a428a6
+oid sha256:4d7e2043e09ec2b328d02f0638a57759f62a1b72350c1a7738b78953d31c6142
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.8962129950523376,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-150",
-  "epoch": 0.10301313417460727,
+  "best_metric": 0.8956434726715088,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-250",
+  "epoch": 0.12876641771825909,
   "eval_steps": 50,
-  "global_step": 200,
+  "global_step": 250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -363,11 +363,100 @@
       "eval_steps_per_second": 0.887,
       "num_input_tokens_seen": 2339304,
       "step": 200
+    },
+    {
+      "epoch": 0.10558846252897244,
+      "grad_norm": 0.6222576114383499,
+      "learning_rate": 9.997103130994296e-05,
+      "loss": 0.9003,
+      "num_input_tokens_seen": 2397808,
+      "step": 205
+    },
+    {
+      "epoch": 0.10816379088333762,
+      "grad_norm": 0.2983149992592585,
+      "learning_rate": 9.996216446216267e-05,
+      "loss": 0.8969,
+      "num_input_tokens_seen": 2456288,
+      "step": 210
+    },
+    {
+      "epoch": 0.11073911923770281,
+      "grad_norm": 0.3505370510576513,
+      "learning_rate": 9.995211600182397e-05,
+      "loss": 0.9114,
+      "num_input_tokens_seen": 2514784,
+      "step": 215
+    },
+    {
+      "epoch": 0.11331444759206799,
+      "grad_norm": 0.3683806652106065,
+      "learning_rate": 9.994088616657444e-05,
+      "loss": 0.899,
+      "num_input_tokens_seen": 2573240,
+      "step": 220
+    },
+    {
+      "epoch": 0.11588977594643317,
+      "grad_norm": 0.21111769827155855,
+      "learning_rate": 9.992847522200133e-05,
+      "loss": 0.898,
+      "num_input_tokens_seen": 2631672,
+      "step": 225
+    },
+    {
+      "epoch": 0.11846510430079835,
+      "grad_norm": 0.3426987181783304,
+      "learning_rate": 9.99148834616253e-05,
+      "loss": 0.9006,
+      "num_input_tokens_seen": 2690112,
+      "step": 230
+    },
+    {
+      "epoch": 0.12104043265516354,
+      "grad_norm": 0.236983209071443,
+      "learning_rate": 9.990011120689351e-05,
+      "loss": 0.8973,
+      "num_input_tokens_seen": 2748608,
+      "step": 235
+    },
+    {
+      "epoch": 0.12361576100952872,
+      "grad_norm": 0.4575208248826409,
+      "learning_rate": 9.988415880717194e-05,
+      "loss": 0.8885,
+      "num_input_tokens_seen": 2807080,
+      "step": 240
+    },
+    {
+      "epoch": 0.1261910893638939,
+      "grad_norm": 0.5470317919414993,
+      "learning_rate": 9.986702663973722e-05,
+      "loss": 0.9066,
+      "num_input_tokens_seen": 2865520,
+      "step": 245
+    },
+    {
+      "epoch": 0.12876641771825909,
+      "grad_norm": 0.4992479706331095,
+      "learning_rate": 9.98487151097676e-05,
+      "loss": 0.9098,
+      "num_input_tokens_seen": 2924016,
+      "step": 250
+    },
+    {
+      "epoch": 0.12876641771825909,
+      "eval_loss": 0.8956434726715088,
+      "eval_runtime": 17.4804,
+      "eval_samples_per_second": 3.432,
+      "eval_steps_per_second": 0.858,
+      "num_input_tokens_seen": 2924016,
+      "step": 250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 2339304,
+  "num_input_tokens_seen": 2924016,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -382,7 +471,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 131317812690944.0,
+  "total_flos": 164152867291136.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null