Training in progress, step 2150, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b39895ee437656d2822d9ef3e148e364db825b03d8467830b69e5610c256e5a
+oid sha256:ff2cc1f16a485d3e450e8a2a866e3d76b5149212ac068f60cacb581a8329cec2
 size 18516456

last-checkpoint/global_step2149/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d227247b6f907687bc60ba64b8533e1fdc775a60d9a690a188c20c4e98289ae8
+size 27700976

last-checkpoint/global_step2149/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29b4630f9d498b66da84fe1dd62e231bd9d6ae711c5584cf1da347de5c6a18bf
+size 27700976

last-checkpoint/global_step2149/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28105d136b8f444e7b65dc83e72fa1fa581f0311a591bb7af21997ad7cdb4956
+size 27700976

last-checkpoint/global_step2149/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d638c4d78d5b4a5a073a1667ea56ffed3aacab81338df8a0c4c81db734fd50f5
+size 27700976

last-checkpoint/global_step2149/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd8089e67d480f0b17a75b0d0d1ff5ebe23bfc1db48bdda3659c12d7beeea256
+size 411571

last-checkpoint/global_step2149/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a144cdd228e1c5b68e61511d761c804238e096259e9ae91194a2da8e14a20eaa
+size 411507

last-checkpoint/global_step2149/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:842afc1739069bba0075155375cc79f6bd1ded3ca74268ff9206f7f43241f74c
+size 411507

last-checkpoint/global_step2149/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de403527d3d9657db7d8d9aba02b7bf9a24a78e391cf51caada74f12dd2ae0cf
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2099
+global_step2149

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b105708e2c99c8661b46698b8ccc5799ac83c1f0fc6a30c2d41c9fbfb349d480
+oid sha256:1c2f72d01585273766959f0cc9805fab753b53f20e581399855a293176ace988
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc44cd2015d8c8fc2f109f07c797876873a52f478c57b0350b8a2cf5dcb17f25
+oid sha256:3fd1ecda2bb159be37a2a23800e098324f5b0334e7189df47c343ca6cb7605a2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9843dec201b5a542ebd69abfc596f99ad5a000cf81dab52c6a2c52a5b9224ea7
+oid sha256:cf71c84ea2995fbc545b918d03f7f94c92293ca2e33343f177e6fd04531b7b19
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2296f0efda653dd4c7e861f5a867baa09d6d8bb50e57bc69af930268b40de9ef
+oid sha256:72c53116f0f4c80841c24cd681d5fbd5a5992b259583a4cfb493f8f3e4544d82
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41ca6ff868655333e43bb2ded3dfe8c66b7ed82fa8a98bace2e730c42bb9959b
+oid sha256:ca5770eed90150126cfd6d4c180a03d91fe366663610f38ec72b635b0b8cfd11
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.0813803759979397,
+  "epoch": 1.1071336595415915,
   "eval_steps": 50,
-  "global_step": 2100,
+  "global_step": 2150,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3745,11 +3745,100 @@
       "eval_steps_per_second": 0.932,
       "num_input_tokens_seen": 24555904,
       "step": 2100
+    },
+    {
+      "epoch": 1.083955704352305,
+      "grad_norm": 5.259882631403194,
+      "learning_rate": 3.4687969674819906e-05,
+      "loss": 0.3827,
+      "num_input_tokens_seen": 24614392,
+      "step": 2105
+    },
+    {
+      "epoch": 1.0865310327066702,
+      "grad_norm": 4.276410371848581,
+      "learning_rate": 3.445667702902969e-05,
+      "loss": 0.3676,
+      "num_input_tokens_seen": 24672848,
+      "step": 2110
+    },
+    {
+      "epoch": 1.0891063610610352,
+      "grad_norm": 10.209040215860048,
+      "learning_rate": 3.4225751985120215e-05,
+      "loss": 0.3253,
+      "num_input_tokens_seen": 24731344,
+      "step": 2115
+    },
+    {
+      "epoch": 1.0916816894154004,
+      "grad_norm": 6.169752493978822,
+      "learning_rate": 3.3995200004502816e-05,
+      "loss": 0.4297,
+      "num_input_tokens_seen": 24789832,
+      "step": 2120
+    },
+    {
+      "epoch": 1.0942570177697657,
+      "grad_norm": 4.238650399680663,
+      "learning_rate": 3.3765026539765834e-05,
+      "loss": 0.3536,
+      "num_input_tokens_seen": 24848264,
+      "step": 2125
+    },
+    {
+      "epoch": 1.0968323461241307,
+      "grad_norm": 5.445173229006411,
+      "learning_rate": 3.3535237034545675e-05,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 24906744,
+      "step": 2130
+    },
+    {
+      "epoch": 1.099407674478496,
+      "grad_norm": 4.508587102151408,
+      "learning_rate": 3.330583692339802e-05,
+      "loss": 0.3666,
+      "num_input_tokens_seen": 24965256,
+      "step": 2135
+    },
+    {
+      "epoch": 1.1019830028328612,
+      "grad_norm": 5.836654544282574,
+      "learning_rate": 3.307683163166934e-05,
+      "loss": 0.3334,
+      "num_input_tokens_seen": 25023768,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1045583311872265,
+      "grad_norm": 6.855334175793522,
+      "learning_rate": 3.284822657536856e-05,
+      "loss": 0.3848,
+      "num_input_tokens_seen": 25082248,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1071336595415915,
+      "grad_norm": 5.3006438448712565,
+      "learning_rate": 3.262002716103897e-05,
+      "loss": 0.3699,
+      "num_input_tokens_seen": 25140752,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1071336595415915,
+      "eval_loss": 0.8523861169815063,
+      "eval_runtime": 16.0023,
+      "eval_samples_per_second": 3.749,
+      "eval_steps_per_second": 0.937,
+      "num_input_tokens_seen": 25140752,
+      "step": 2150
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 24555904,
+  "num_input_tokens_seen": 25140752,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3764,7 +3853,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1378926060634112.0,
+  "total_flos": 1411768699584512.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null