Training in progress, step 2500, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7308e6252b959616b2fe7c9a1e30cbe6a34ffb2d029f50db793552fbd2bdbfbc
+oid sha256:3a6f08fab5c9cd3e3824141f661d04fab419e16f0f27aa8be427a55a045a5024
 size 577789320

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e080c2a47685ca953b3b9d5095ee4227f7d627920903f3549b23b13a6c86e24
+oid sha256:59d2fcaa714da200edbccb39a9f2728d1eac302590a189b7a1ad33e51ddded74
 size 1155772233

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f01983801a45234478c13db780514a852cdeaff2aa79f279442e47dc68cb11d
+oid sha256:e0a3df38c17b475ea7e47b6c462cb3838a91cdec2a3f47a69fc35d66481cfa21
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15b560a9dc43b2ed5d8c7b0910cf19c12068f6e2db4cd26fd270e940d4d1787b
+oid sha256:b382f91d6a62c99969921e0d8014a3b1f89a198a6a81ab888e1194bc144c13d5
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.36490994691848755,
-  "best_model_checkpoint": "mikhail_panzo/zlm_b128_le4_s8000/checkpoint-2000",
-  "epoch": 3.350785340314136,
+  "best_metric": 0.3528364896774292,
+  "best_model_checkpoint": "mikhail_panzo/zlm_b128_le4_s8000/checkpoint-2500",
+  "epoch": 4.18848167539267,
   "eval_steps": 500,
-  "global_step": 2000,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -319,6 +319,84 @@
       "eval_samples_per_second": 31.924,
       "eval_steps_per_second": 3.994,
       "step": 2000
+    },
+    {
+      "epoch": 3.4345549738219896,
+      "grad_norm": 2.121384859085083,
+      "learning_rate": 9.918333333333334e-05,
+      "loss": 0.4058,
+      "step": 2050
+    },
+    {
+      "epoch": 3.518324607329843,
+      "grad_norm": 1.645984411239624,
+      "learning_rate": 9.835e-05,
+      "loss": 0.4021,
+      "step": 2100
+    },
+    {
+      "epoch": 3.6020942408376966,
+      "grad_norm": 1.246239185333252,
+      "learning_rate": 9.751666666666666e-05,
+      "loss": 0.3991,
+      "step": 2150
+    },
+    {
+      "epoch": 3.6858638743455496,
+      "grad_norm": 1.9096795320510864,
+      "learning_rate": 9.668333333333334e-05,
+      "loss": 0.3961,
+      "step": 2200
+    },
+    {
+      "epoch": 3.769633507853403,
+      "grad_norm": 1.8867601156234741,
+      "learning_rate": 9.585000000000001e-05,
+      "loss": 0.3904,
+      "step": 2250
+    },
+    {
+      "epoch": 3.8534031413612566,
+      "grad_norm": 1.7438101768493652,
+      "learning_rate": 9.501666666666668e-05,
+      "loss": 0.3895,
+      "step": 2300
+    },
+    {
+      "epoch": 3.93717277486911,
+      "grad_norm": 1.1799490451812744,
+      "learning_rate": 9.418333333333334e-05,
+      "loss": 0.4027,
+      "step": 2350
+    },
+    {
+      "epoch": 4.020942408376963,
+      "grad_norm": 1.1952763795852661,
+      "learning_rate": 9.335e-05,
+      "loss": 0.3893,
+      "step": 2400
+    },
+    {
+      "epoch": 4.104712041884817,
+      "grad_norm": 2.008756160736084,
+      "learning_rate": 9.251666666666667e-05,
+      "loss": 0.3878,
+      "step": 2450
+    },
+    {
+      "epoch": 4.18848167539267,
+      "grad_norm": 2.2693591117858887,
+      "learning_rate": 9.168333333333333e-05,
+      "loss": 0.3863,
+      "step": 2500
+    },
+    {
+      "epoch": 4.18848167539267,
+      "eval_loss": 0.3528364896774292,
+      "eval_runtime": 272.7627,
+      "eval_samples_per_second": 31.122,
+      "eval_steps_per_second": 3.893,
+      "step": 2500
     }
   ],
   "logging_steps": 50,
@@ -338,7 +416,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.5804068604023104e+16,
+  "total_flos": 4.477753664307475e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null