Training in progress, step 2500, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca029c96d767da45b7323e47c97d5e6f73b5c0d8dc25a679391682e01d0116f3
+oid sha256:b6594507d2093f7d04ea48dba9b17f445e9541e34c9ab8598b25c91de8e5ab89
 size 577789320

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc2a528c117c0804a8911914e1d71117feb3539aa52b1dcd1672a20000241a5c
+oid sha256:fe7a50c86b765550303567a7c71dc3287d2a1793b843374f3662606fefc4c7b7
 size 1155772233

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1408fe68df18c0e59db68d59ba316117312bd780df72ef23ec45a7e9f2b2bcd9
+oid sha256:6e8aa2535c3a1ec264ce2c31a7ac416d4861404181dd39ae479182880849bcf2
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f32a07717606d664792d4ebfd434223710fe948a637ff5f34234da98aa96ac43
+oid sha256:d8452e8180915c2a4933789804dc213986b2971ca2876fd0285cc8294b6a4c56
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.4116212725639343,
-  "best_model_checkpoint": "mikhail-panzo/ceb_b64_le5_s8000/checkpoint-1500",
-  "epoch": 58.8235294117647,
+  "best_metric": 0.3997121751308441,
+  "best_model_checkpoint": "mikhail-panzo/ceb_b64_le5_s8000/checkpoint-2500",
+  "epoch": 98.03921568627452,
   "eval_steps": 500,
-  "global_step": 1500,
+  "global_step": 2500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -241,6 +241,162 @@
       "eval_samples_per_second": 28.665,
       "eval_steps_per_second": 3.663,
       "step": 1500
+    },
+    {
+      "epoch": 60.78431372549019,
+      "grad_norm": 0.9767690896987915,
+      "learning_rate": 7.745e-06,
+      "loss": 0.4517,
+      "step": 1550
+    },
+    {
+      "epoch": 62.745098039215684,
+      "grad_norm": 1.4193260669708252,
+      "learning_rate": 7.995e-06,
+      "loss": 0.4537,
+      "step": 1600
+    },
+    {
+      "epoch": 64.70588235294117,
+      "grad_norm": 1.3294360637664795,
+      "learning_rate": 8.245000000000002e-06,
+      "loss": 0.4435,
+      "step": 1650
+    },
+    {
+      "epoch": 66.66666666666667,
+      "grad_norm": 0.8386899828910828,
+      "learning_rate": 8.495e-06,
+      "loss": 0.4507,
+      "step": 1700
+    },
+    {
+      "epoch": 68.62745098039215,
+      "grad_norm": 1.0917119979858398,
+      "learning_rate": 8.745000000000002e-06,
+      "loss": 0.4409,
+      "step": 1750
+    },
+    {
+      "epoch": 70.58823529411765,
+      "grad_norm": 1.0725489854812622,
+      "learning_rate": 8.995000000000001e-06,
+      "loss": 0.4449,
+      "step": 1800
+    },
+    {
+      "epoch": 72.54901960784314,
+      "grad_norm": 1.3506999015808105,
+      "learning_rate": 9.245e-06,
+      "loss": 0.4496,
+      "step": 1850
+    },
+    {
+      "epoch": 74.50980392156863,
+      "grad_norm": 0.9701379537582397,
+      "learning_rate": 9.495000000000001e-06,
+      "loss": 0.4384,
+      "step": 1900
+    },
+    {
+      "epoch": 76.47058823529412,
+      "grad_norm": 1.7079219818115234,
+      "learning_rate": 9.745e-06,
+      "loss": 0.4374,
+      "step": 1950
+    },
+    {
+      "epoch": 78.43137254901961,
+      "grad_norm": 1.87998628616333,
+      "learning_rate": 9.995000000000002e-06,
+      "loss": 0.4346,
+      "step": 2000
+    },
+    {
+      "epoch": 78.43137254901961,
+      "eval_loss": 0.4027920663356781,
+      "eval_runtime": 6.3546,
+      "eval_samples_per_second": 28.326,
+      "eval_steps_per_second": 3.619,
+      "step": 2000
+    },
+    {
+      "epoch": 80.3921568627451,
+      "grad_norm": 1.1510419845581055,
+      "learning_rate": 9.918333333333335e-06,
+      "loss": 0.4326,
+      "step": 2050
+    },
+    {
+      "epoch": 82.3529411764706,
+      "grad_norm": 1.2605654001235962,
+      "learning_rate": 9.835000000000002e-06,
+      "loss": 0.4355,
+      "step": 2100
+    },
+    {
+      "epoch": 84.31372549019608,
+      "grad_norm": 0.866606593132019,
+      "learning_rate": 9.751666666666667e-06,
+      "loss": 0.4286,
+      "step": 2150
+    },
+    {
+      "epoch": 86.27450980392157,
+      "grad_norm": 2.0733227729797363,
+      "learning_rate": 9.668333333333334e-06,
+      "loss": 0.4365,
+      "step": 2200
+    },
+    {
+      "epoch": 88.23529411764706,
+      "grad_norm": 0.9726402759552002,
+      "learning_rate": 9.585e-06,
+      "loss": 0.4367,
+      "step": 2250
+    },
+    {
+      "epoch": 90.19607843137256,
+      "grad_norm": 1.0713222026824951,
+      "learning_rate": 9.501666666666667e-06,
+      "loss": 0.4288,
+      "step": 2300
+    },
+    {
+      "epoch": 92.15686274509804,
+      "grad_norm": 1.5218483209609985,
+      "learning_rate": 9.418333333333334e-06,
+      "loss": 0.435,
+      "step": 2350
+    },
+    {
+      "epoch": 94.11764705882354,
+      "grad_norm": 0.8391968011856079,
+      "learning_rate": 9.335000000000001e-06,
+      "loss": 0.431,
+      "step": 2400
+    },
+    {
+      "epoch": 96.07843137254902,
+      "grad_norm": 1.3989890813827515,
+      "learning_rate": 9.251666666666668e-06,
+      "loss": 0.4251,
+      "step": 2450
+    },
+    {
+      "epoch": 98.03921568627452,
+      "grad_norm": 0.9168123006820679,
+      "learning_rate": 9.168333333333333e-06,
+      "loss": 0.4292,
+      "step": 2500
+    },
+    {
+      "epoch": 98.03921568627452,
+      "eval_loss": 0.3997121751308441,
+      "eval_runtime": 6.4036,
+      "eval_samples_per_second": 28.109,
+      "eval_steps_per_second": 3.592,
+      "step": 2500
     }
   ],
   "logging_steps": 50,
@@ -260,7 +416,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.6328687985209952e+16,
+  "total_flos": 2.7204142566350376e+16,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null