Training in progress, step 1500, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec22c3d9bcd6353319973a9b948e015bf2b55e5b016dd4a047ac623b8d75a936
+oid sha256:0b3878b2837c54460760b7689511b1b119ade195a40bcc1bd58d5a2fa1c333cc
 size 577789320

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc8ad9821b5b1d9d37cd3a4a43587f5f9ffc47c277706630426b4c035c536e57
+oid sha256:15856761cc33aa3ef4082afef8bcd92b764bb43930646ef8255717cfd74acefa
 size 1155772233

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a010ad19aaaa343fbfbf6d1ce05dfd1e56fb0ac07188b993fc06e025d81ccc1
+oid sha256:dc65651f7b378ef8fe4da6f30ced75150f8b4133aff002554882ce4b7f48b94e
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8210f8e2249c280ac4965b4cc059199e79e619c33beb58bb250012029c0aaadf
+oid sha256:1a94e74103068a5fb8b31be8524087df4f57bd43f07d1db2b9d36084882fdbb5
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.402164101600647,
-  "best_model_checkpoint": "mikhail_panzo/zlm_b128_le4_s8000/checkpoint-1000",
-  "epoch": 1.675392670157068,
+  "best_metric": 0.3853827118873596,
+  "best_model_checkpoint": "mikhail_panzo/zlm_b128_le4_s8000/checkpoint-1500",
+  "epoch": 2.513089005235602,
   "eval_steps": 500,
-  "global_step": 1000,
+  "global_step": 1500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -163,6 +163,84 @@
       "eval_samples_per_second": 31.959,
       "eval_steps_per_second": 3.998,
       "step": 1000
+    },
+    {
+      "epoch": 1.7591623036649215,
+      "grad_norm": 4.97900390625,
+      "learning_rate": 5.245e-05,
+      "loss": 0.4553,
+      "step": 1050
+    },
+    {
+      "epoch": 1.8429319371727748,
+      "grad_norm": 1.9889676570892334,
+      "learning_rate": 5.495e-05,
+      "loss": 0.449,
+      "step": 1100
+    },
+    {
+      "epoch": 1.9267015706806283,
+      "grad_norm": 1.5135546922683716,
+      "learning_rate": 5.745e-05,
+      "loss": 0.4353,
+      "step": 1150
+    },
+    {
+      "epoch": 2.0104712041884816,
+      "grad_norm": 7.610673904418945,
+      "learning_rate": 5.995000000000001e-05,
+      "loss": 0.4311,
+      "step": 1200
+    },
+    {
+      "epoch": 2.094240837696335,
+      "grad_norm": 2.049562454223633,
+      "learning_rate": 6.245000000000001e-05,
+      "loss": 0.4312,
+      "step": 1250
+    },
+    {
+      "epoch": 2.1780104712041886,
+      "grad_norm": 1.4102027416229248,
+      "learning_rate": 6.494999999999999e-05,
+      "loss": 0.4282,
+      "step": 1300
+    },
+    {
+      "epoch": 2.261780104712042,
+      "grad_norm": 1.701119065284729,
+      "learning_rate": 6.745e-05,
+      "loss": 0.4272,
+      "step": 1350
+    },
+    {
+      "epoch": 2.345549738219895,
+      "grad_norm": 2.0149667263031006,
+      "learning_rate": 6.995e-05,
+      "loss": 0.4277,
+      "step": 1400
+    },
+    {
+      "epoch": 2.4293193717277486,
+      "grad_norm": 2.1658883094787598,
+      "learning_rate": 7.245000000000001e-05,
+      "loss": 0.4247,
+      "step": 1450
+    },
+    {
+      "epoch": 2.513089005235602,
+      "grad_norm": 2.6821463108062744,
+      "learning_rate": 7.495e-05,
+      "loss": 0.4169,
+      "step": 1500
+    },
+    {
+      "epoch": 2.513089005235602,
+      "eval_loss": 0.3853827118873596,
+      "eval_runtime": 269.2607,
+      "eval_samples_per_second": 31.527,
+      "eval_steps_per_second": 3.944,
+      "step": 1500
     }
   ],
   "logging_steps": 50,
@@ -182,7 +260,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.791435708710208e+16,
+  "total_flos": 2.686672014814656e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null