Training in progress, step 1850, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:703d6c7e98f4bf68731cde74c5b20cc8e4bb82cbcfea9af1822815e13d9fd381
+oid sha256:08913ca116f96937bc81ec7e7fb392702bac2ed46bbaaf9d801c2159669cfd36
 size 29034840

last-checkpoint/global_step1850/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54e9bb7b8347c841ec0408e1309b7c0726e4f724216c4ea6c35469797687805
+size 43429616

last-checkpoint/global_step1850/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9513de97adc66cd1fc8e05d8f2861dada0cc6abe0c5eef5eaff0201912909ce
+size 43429616

last-checkpoint/global_step1850/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b413dd4a368c428030156eba96a1c201eb85fb7f6e587b72ae71e81136f788
+size 43429616

last-checkpoint/global_step1850/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a1b8d18ccdef75fe5b06f92bab56da9d7843ea61986f56f9152154488552b6
+size 43429616

last-checkpoint/global_step1850/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae68137480686f51324164cfe12b894960f5dac924d517328b9a25cc5475cb52
+size 637299

last-checkpoint/global_step1850/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75d6f58ca4c8da0e147329d82ed7539d567f755227e39ad9cc5e35d551f74abd
+size 637171

last-checkpoint/global_step1850/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e205dd3e9640f3af3ad21d7b533feac51eb10ed278346cf338f88fc343a8131d
+size 637171

last-checkpoint/global_step1850/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4c50db8340941b9c0ee37b041dc49397ad831d065dc3058b939ae5d78b2dec
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1800
+global_step1850

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d76f093328c54f2b94a10b8b50dc92fc99ceff9e3949d050a70042526f1d0eb
+oid sha256:7cc13c69d2b97530a3e18634e2f473678ea6880a6b34244c9c86a457f70137e1
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54b172dd00cba9f005761d85fb0804f751caf6e1cc8294d1873354a890cb9909
+oid sha256:fcfd90610e35a8f40098e1413d3e3a1658276c859045d3450f227cab64c9081a
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec6adef733bd7630aa48eff1a6edaabc275d67293dcb0b7a64d71451405d489d
+oid sha256:663ae184d05dd950d27e1f419e0306ca3b141b9d7f075a80804cada62f64a363
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ce1c4ba1932c0f698bef90e0e4e74ded1e0db5fc35282a0815899b8be759e67
+oid sha256:ec4f0a72927b3837a0e73cb5612acf4318c503f3ba2108b3925f750332a49b60
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df430b942be14b68acaf59a6a16e4c1e563f436ec4bae0ac022a9110d0fbea5b
+oid sha256:93b576cb2a6641fdd8886c6f5607f7b3729a9eb185add7e75756a0fd0af07d89
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.28714123368263245,
   "best_model_checkpoint": "saves/CADICA_qwenvl_direction_then_DetectAndClassify_scale6/lora/sft/checkpoint-1550",
-  "epoch": 0.5317577548005908,
+  "epoch": 0.5465288035450517,
   "eval_steps": 50,
-  "global_step": 1800,
+  "global_step": 1850,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3211,11 +3211,100 @@
       "eval_steps_per_second": 0.776,
       "num_input_tokens_seen": 18691928,
       "step": 1800
+    },
+    {
+      "epoch": 0.5332348596750369,
+      "grad_norm": 2.1114949572134596,
+      "learning_rate": 8.771686672554067e-05,
+      "loss": 0.3145,
+      "num_input_tokens_seen": 18743600,
+      "step": 1805
+    },
+    {
+      "epoch": 0.534711964549483,
+      "grad_norm": 2.661731170516522,
+      "learning_rate": 8.763657956631462e-05,
+      "loss": 0.3207,
+      "num_input_tokens_seen": 18794920,
+      "step": 1810
+    },
+    {
+      "epoch": 0.5361890694239291,
+      "grad_norm": 10.01115206187178,
+      "learning_rate": 8.75560678676323e-05,
+      "loss": 0.2463,
+      "num_input_tokens_seen": 18847600,
+      "step": 1815
+    },
+    {
+      "epoch": 0.5376661742983752,
+      "grad_norm": 2.5996909028595963,
+      "learning_rate": 8.747533210982575e-05,
+      "loss": 0.3199,
+      "num_input_tokens_seen": 18899936,
+      "step": 1820
+    },
+    {
+      "epoch": 0.5391432791728212,
+      "grad_norm": 1.5344678453995588,
+      "learning_rate": 8.739437277456366e-05,
+      "loss": 0.282,
+      "num_input_tokens_seen": 18952336,
+      "step": 1825
+    },
+    {
+      "epoch": 0.5406203840472673,
+      "grad_norm": 1.0236614886630173,
+      "learning_rate": 8.731319034484862e-05,
+      "loss": 0.2846,
+      "num_input_tokens_seen": 19004528,
+      "step": 1830
+    },
+    {
+      "epoch": 0.5420974889217134,
+      "grad_norm": 2.459552261348287,
+      "learning_rate": 8.723178530501417e-05,
+      "loss": 0.3537,
+      "num_input_tokens_seen": 19056296,
+      "step": 1835
+    },
+    {
+      "epoch": 0.5435745937961596,
+      "grad_norm": 6.813862731810714,
+      "learning_rate": 8.7150158140722e-05,
+      "loss": 0.3289,
+      "num_input_tokens_seen": 19108832,
+      "step": 1840
+    },
+    {
+      "epoch": 0.5450516986706057,
+      "grad_norm": 9.828132924879462,
+      "learning_rate": 8.706830933895894e-05,
+      "loss": 0.3094,
+      "num_input_tokens_seen": 19160960,
+      "step": 1845
+    },
+    {
+      "epoch": 0.5465288035450517,
+      "grad_norm": 1.3290993068913195,
+      "learning_rate": 8.698623938803409e-05,
+      "loss": 0.2828,
+      "num_input_tokens_seen": 19212992,
+      "step": 1850
+    },
+    {
+      "epoch": 0.5465288035450517,
+      "eval_loss": 0.3780718147754669,
+      "eval_runtime": 19.3274,
+      "eval_samples_per_second": 3.104,
+      "eval_steps_per_second": 0.776,
+      "num_input_tokens_seen": 19212992,
+      "step": 1850
     }
   ],
   "logging_steps": 5,
   "max_steps": 6770,
-  "num_input_tokens_seen": 18691928,
+  "num_input_tokens_seen": 19212992,
   "num_train_epochs": 2,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3230,7 +3319,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1233113797361664.0,
+  "total_flos": 1267532503711744.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null