Training in progress, step 3500, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30d951c379da9e40e925d0e9823bfafd91b398b9c593f0fbd9f7aacf86d39488
+oid sha256:9bfa6fd540e99d4d835ccb5f43e5cbd0a121e435e68bf6b732f387076da8f095
 size 577789320

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e50dd72dbdf52c58ab0a34685c567d0a859e2ac19bb2d944204af4086178fa28
+oid sha256:b607d5b8da092de2116b5787d176adc23cff3ca8a5fd2f0d363db75ecb5424cc
 size 1155772233

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:778b3eb7f69f0b8d1febc7d7292dba969822e2d3a8eb043627219091aed28184
+oid sha256:c5773d34f1b56b34758c0948a73d049e397effcdb1da51deddbe31d1812bd4e6
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe9df1d7779a957d8857092ecc09a3b38d73d9d62600477c7a2f5f4fc70c502b
+oid sha256:deac897f650af2681d7103c3cb2fd69164944cc63ad83c3b43ad7d1ddd3c29fb
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.41350311040878296,
-  "best_model_checkpoint": "mikhail_panzo/zlm-fil_b64_le5_s8000/checkpoint-3000",
-  "epoch": 130.43478260869566,
+  "best_metric": 0.4111216962337494,
+  "best_model_checkpoint": "mikhail_panzo/zlm-fil_b64_le5_s8000/checkpoint-3500",
+  "epoch": 152.17391304347825,
   "eval_steps": 500,
-  "global_step": 3000,
+  "global_step": 3500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -475,6 +475,84 @@
       "eval_samples_per_second": 12.572,
       "eval_steps_per_second": 1.64,
       "step": 3000
+    },
+    {
+      "epoch": 132.6086956521739,
+      "grad_norm": 1.8065855503082275,
+      "learning_rate": 8.251666666666668e-06,
+      "loss": 0.4361,
+      "step": 3050
+    },
+    {
+      "epoch": 134.7826086956522,
+      "grad_norm": 0.8073704838752747,
+      "learning_rate": 8.168333333333334e-06,
+      "loss": 0.4339,
+      "step": 3100
+    },
+    {
+      "epoch": 136.95652173913044,
+      "grad_norm": 1.2890065908432007,
+      "learning_rate": 8.085000000000001e-06,
+      "loss": 0.4325,
+      "step": 3150
+    },
+    {
+      "epoch": 139.1304347826087,
+      "grad_norm": 1.336401104927063,
+      "learning_rate": 8.001666666666668e-06,
+      "loss": 0.4334,
+      "step": 3200
+    },
+    {
+      "epoch": 141.30434782608697,
+      "grad_norm": 1.2965891361236572,
+      "learning_rate": 7.918333333333333e-06,
+      "loss": 0.4298,
+      "step": 3250
+    },
+    {
+      "epoch": 143.47826086956522,
+      "grad_norm": 0.8761409521102905,
+      "learning_rate": 7.835e-06,
+      "loss": 0.4231,
+      "step": 3300
+    },
+    {
+      "epoch": 145.65217391304347,
+      "grad_norm": 1.1475930213928223,
+      "learning_rate": 7.751666666666667e-06,
+      "loss": 0.4316,
+      "step": 3350
+    },
+    {
+      "epoch": 147.82608695652175,
+      "grad_norm": 0.8305974006652832,
+      "learning_rate": 7.668333333333334e-06,
+      "loss": 0.4277,
+      "step": 3400
+    },
+    {
+      "epoch": 150.0,
+      "grad_norm": 1.6335935592651367,
+      "learning_rate": 7.585e-06,
+      "loss": 0.4248,
+      "step": 3450
+    },
+    {
+      "epoch": 152.17391304347825,
+      "grad_norm": 1.1171984672546387,
+      "learning_rate": 7.501666666666667e-06,
+      "loss": 0.4318,
+      "step": 3500
+    },
+    {
+      "epoch": 152.17391304347825,
+      "eval_loss": 0.4111216962337494,
+      "eval_runtime": 13.0991,
+      "eval_samples_per_second": 12.291,
+      "eval_steps_per_second": 1.603,
+      "step": 3500
     }
   ],
   "logging_steps": 50,
@@ -494,7 +572,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.635834796240768e+16,
+  "total_flos": 4.241687452614216e+16,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null