Training in progress, step 2200, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff2cc1f16a485d3e450e8a2a866e3d76b5149212ac068f60cacb581a8329cec2
+oid sha256:76b20ab8c2a7403c32454801b8a1cf7e477efa58783a51bc7e3abf420b274c08
 size 18516456

last-checkpoint/global_step2199/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b0613c004ef61ba46c469e7988afd6177f0d8ff04ff8ee6444bdd1d1fc9108c
+size 27700976

last-checkpoint/global_step2199/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b3dba5ff896b70830e8073bb54f1ef435a0fa45b55ce17454c205623c4e48fb
+size 27700976

last-checkpoint/global_step2199/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d68c67d4836968cea589a6fddd1c8dbd6f70c5a0af74ae71adda19649d8531b
+size 27700976

last-checkpoint/global_step2199/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:173cf728f5217ccf79a2d5628b083113384f213ca6fb71100b406e577b906625
+size 27700976

last-checkpoint/global_step2199/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c88274e48eccf85cd6d66b4ac47fffddeff427b70680bfa735d8d1c8a4351cd
+size 411571

last-checkpoint/global_step2199/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17b85ef590965c58493d34cf18fbce75161a74838c86d524eb7fa17e17fd05ac
+size 411507

last-checkpoint/global_step2199/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4779fdbd87e10a9b125e08f201524dc4bce730d2f09d576123faf2f33a8445e
+size 411507

last-checkpoint/global_step2199/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2980e5a9feb469339dd5196c24deea3e2fd2c3442acf2f2a882e7f13b04409e8
+size 411507

last-checkpoint/latest

@@ -1 +1 @@
-global_step2149
+global_step2199

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c2f72d01585273766959f0cc9805fab753b53f20e581399855a293176ace988
+oid sha256:3187a61ccc2722c440dc24ae4a6eefe6b9e5daccf9e92473bbb4483c7751ea77
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fd1ecda2bb159be37a2a23800e098324f5b0334e7189df47c343ca6cb7605a2
+oid sha256:b0f2a0df922fb3337cf2562745ebe8d5adf433ca45cb4e3da33a21b48183c000
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf71c84ea2995fbc545b918d03f7f94c92293ca2e33343f177e6fd04531b7b19
+oid sha256:f4d84b5276f687f44c9af60b1e41cd7b93a6d1659e36831a7bc021b5635d663b
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72c53116f0f4c80841c24cd681d5fbd5a5992b259583a4cfb493f8f3e4544d82
+oid sha256:7d69159433c88b97106cf21b92eb5a3f66f0c826aa268d82a47b3faed1ac86cd
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca5770eed90150126cfd6d4c180a03d91fe366663610f38ec72b635b0b8cfd11
+oid sha256:6225488c9a450b7edfa6b28ac40ecd217bccdb84073c98a64aefcefa7ee337d2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.6319106221199036,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_classily_scale4_frozenVision/lora/sft/checkpoint-1600",
-  "epoch": 1.1071336595415915,
+  "epoch": 1.1328869430852433,
   "eval_steps": 50,
-  "global_step": 2150,
+  "global_step": 2200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3834,11 +3834,100 @@
       "eval_steps_per_second": 0.937,
       "num_input_tokens_seen": 25140752,
       "step": 2150
+    },
+    {
+      "epoch": 1.1097089878959567,
+      "grad_norm": 3.943124296473041,
+      "learning_rate": 3.2392238785630386e-05,
+      "loss": 0.3154,
+      "num_input_tokens_seen": 25199208,
+      "step": 2155
+    },
+    {
+      "epoch": 1.112284316250322,
+      "grad_norm": 8.398532132538953,
+      "learning_rate": 3.216486683637146e-05,
+      "loss": 0.3915,
+      "num_input_tokens_seen": 25257680,
+      "step": 2160
+    },
+    {
+      "epoch": 1.114859644604687,
+      "grad_norm": 4.081633194377614,
+      "learning_rate": 3.1937916690642356e-05,
+      "loss": 0.3675,
+      "num_input_tokens_seen": 25316200,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1174349729590523,
+      "grad_norm": 6.920842495491902,
+      "learning_rate": 3.1711393715847476e-05,
+      "loss": 0.4047,
+      "num_input_tokens_seen": 25374656,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1200103013134175,
+      "grad_norm": 8.460113153700512,
+      "learning_rate": 3.14853032692886e-05,
+      "loss": 0.4155,
+      "num_input_tokens_seen": 25433168,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1225856296677827,
+      "grad_norm": 9.825074199159944,
+      "learning_rate": 3.125965069803811e-05,
+      "loss": 0.3966,
+      "num_input_tokens_seen": 25491664,
+      "step": 2180
+    },
+    {
+      "epoch": 1.1251609580221478,
+      "grad_norm": 5.732206927543506,
+      "learning_rate": 3.103444133881261e-05,
+      "loss": 0.3068,
+      "num_input_tokens_seen": 25550128,
+      "step": 2185
+    },
+    {
+      "epoch": 1.127736286376513,
+      "grad_norm": 6.135036052058211,
+      "learning_rate": 3.080968051784666e-05,
+      "loss": 0.386,
+      "num_input_tokens_seen": 25608624,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1303116147308783,
+      "grad_norm": 3.31420885852192,
+      "learning_rate": 3.058537355076683e-05,
+      "loss": 0.3898,
+      "num_input_tokens_seen": 25667128,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1328869430852433,
+      "grad_norm": 8.182546413863832,
+      "learning_rate": 3.0361525742465973e-05,
+      "loss": 0.4016,
+      "num_input_tokens_seen": 25725560,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1328869430852433,
+      "eval_loss": 0.7534744143486023,
+      "eval_runtime": 15.969,
+      "eval_samples_per_second": 3.757,
+      "eval_steps_per_second": 0.939,
+      "num_input_tokens_seen": 25725560,
+      "step": 2200
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 25140752,
+  "num_input_tokens_seen": 25725560,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3853,7 +3942,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1411768699584512.0,
+  "total_flos": 1444609144651776.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null