Training in progress, step 1550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6289479e907da413d0601e67681f7846ac9804602b3a5c4cb53469f8420fd060
+oid sha256:364d970c0898c9aa3378ce697d8c156c845bc89c410b474424ed0f6f28d86575
 size 29034840

last-checkpoint/global_step1550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7c63ce73c181c058cafa2b4d508a1395864a9a6424b52dbc78676e09c1ccfa8
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfdd6837342bded7f7d0aa862755300984a5856947423da899ca167c99d5ca45
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e759f742b69708dc76c5ee53fea0b3b3f48117733c1dbf0b89712caeca10342b
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52d32f3120c36db3f75ebabe7a300d9572bc864b2d59870f72dfdaa3cee01880
+size 43429616

last-checkpoint/global_step1550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a43dd54cbbc70405ab57738adb3ec4581722d997b526b922cea67d12d7a3154
+size 637299

last-checkpoint/global_step1550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2160593af1ce08c3f12b9b90e1c3cbef9f94fa69d7b34c91af296952a4c32634
+size 637171

last-checkpoint/global_step1550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a26b6fdb593b50a1684d8681f4281e3d55be3afd1e2403c04c5de137e4315716
+size 637171

last-checkpoint/global_step1550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4062e6de2aea9cf48856d37c0a6068948c993876856dc4d6c5e69e86ba09a28e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1500
+global_step1550

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd3566049ad1f65d2f434d990deb65584d2b2dcb1aac8e89c68ea37dc533eab7
+oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f96a47dc4176412bc893ccb49c004c8fa1cc8c306d67689d87ed20944233c62
+oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebd9fa3db1079ccf750b71f4eeedbf1f04422fc748026a4b866afe133f9fbfd1
+oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:383e9e252cc8292eef0120c964bdc8033972e800c085c97c42af97379e6b4b5c
+oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f31de0059cfffab00767f1a3eadee4b6350e0f39836feff5d5d9263e375ce9aa
+oid sha256:ff20f7ad4c3f084d2ca3cdf144a9c716a7288c4cbc23f49744772554732ab7ea
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.2966395914554596,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1200",
-  "epoch": 0.4431314623338257,
+  "best_metric": 0.28714123368263245,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1550",
+  "epoch": 0.45790251107828656,
   "eval_steps": 50,
-  "global_step": 1500,
+  "global_step": 1550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2677,11 +2677,100 @@
       "eval_steps_per_second": 0.779,
       "num_input_tokens_seen": 15572048,
       "step": 1500
+    },
+    {
+      "epoch": 0.4446085672082718,
+      "grad_norm": 1.4684583536766802,
+      "learning_rate": 9.210584040403793e-05,
+      "loss": 0.3622,
+      "num_input_tokens_seen": 15623400,
+      "step": 1505
+    },
+    {
+      "epoch": 0.4460856720827179,
+      "grad_norm": 14.96137801867225,
+      "learning_rate": 9.20398524147596e-05,
+      "loss": 0.3192,
+      "num_input_tokens_seen": 15676712,
+      "step": 1510
+    },
+    {
+      "epoch": 0.44756277695716395,
+      "grad_norm": 6.400159478959921,
+      "learning_rate": 9.197361361614339e-05,
+      "loss": 0.3463,
+      "num_input_tokens_seen": 15729304,
+      "step": 1515
+    },
+    {
+      "epoch": 0.44903988183161003,
+      "grad_norm": 4.316336693336091,
+      "learning_rate": 9.190712440336928e-05,
+      "loss": 0.3675,
+      "num_input_tokens_seen": 15780144,
+      "step": 1520
+    },
+    {
+      "epoch": 0.4505169867060561,
+      "grad_norm": 3.3824956383811586,
+      "learning_rate": 9.184038517311126e-05,
+      "loss": 0.3305,
+      "num_input_tokens_seen": 15832032,
+      "step": 1525
+    },
+    {
+      "epoch": 0.4519940915805022,
+      "grad_norm": 8.862997235708779,
+      "learning_rate": 9.177339632353492e-05,
+      "loss": 0.3817,
+      "num_input_tokens_seen": 15884152,
+      "step": 1530
+    },
+    {
+      "epoch": 0.4534711964549483,
+      "grad_norm": 8.942655459138996,
+      "learning_rate": 9.170615825429502e-05,
+      "loss": 0.3742,
+      "num_input_tokens_seen": 15935688,
+      "step": 1535
+    },
+    {
+      "epoch": 0.4549483013293944,
+      "grad_norm": 29.75871004929942,
+      "learning_rate": 9.163867136653327e-05,
+      "loss": 0.34,
+      "num_input_tokens_seen": 15987800,
+      "step": 1540
+    },
+    {
+      "epoch": 0.4564254062038405,
+      "grad_norm": 2.7907856781115816,
+      "learning_rate": 9.157093606287572e-05,
+      "loss": 0.3389,
+      "num_input_tokens_seen": 16039472,
+      "step": 1545
+    },
+    {
+      "epoch": 0.45790251107828656,
+      "grad_norm": 17.658376604188195,
+      "learning_rate": 9.150295274743053e-05,
+      "loss": 0.3588,
+      "num_input_tokens_seen": 16091960,
+      "step": 1550
+    },
+    {
+      "epoch": 0.45790251107828656,
+      "eval_loss": 0.28714123368263245,
+      "eval_runtime": 19.0565,
+      "eval_samples_per_second": 3.149,
+      "eval_steps_per_second": 0.787,
+      "num_input_tokens_seen": 16091960,
+      "step": 1550
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 15572048,
+  "num_input_tokens_seen": 16091960,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2696,7 +2785,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1027290982514688.0,
+  "total_flos": 1061592778407936.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null