Training in progress, step 4000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f37df316feb4858f2a440118f10c706a92a144096e75b10321afd1c2e0cc6f94
+oid sha256:8f0f8979da33f748917d049cc5ff11a0545fa1c5d17f233f6f3c1ec1146a9914
 size 497780352

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0027210dfbc20af719b86b497c1d99b69cc8dd187bd6e90ecd9da520fef05f0
+oid sha256:0bbeda76dd05ca37fe45ede5f34534dd782ec62fe3c26ad951df6b0614b972ad
 size 995617914

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75596e794538906fc3dbf131d0568e28de9e681be900571db24920286dbfef63
+oid sha256:8ff133e609b5f5bc20eed4a2e7ac89b3c0a8b88a47081e6214c0ce30a0d557e2
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25ae61e837be3ada9de530496f3f4fe1da31f4699408459fe2a0b05c3e210f0d
+oid sha256:ef451fce60281d5913091edc9ce429895db044ce88d47b2d438038fa76c50883
 size 1064

last-checkpoint/trainer_state.json

@@ -1,290 +1,236 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 4.0,
   "eval_steps": 200,
-  "global_step": 5000,
+  "global_step": 4000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.2,
-      "eval_loss": 0.07517029345035553,
-      "eval_runtime": 37.992,
-      "eval_samples_per_second": 421.142,
-      "eval_steps_per_second": 6.58,
+      "eval_loss": 0.03862990066409111,
+      "eval_runtime": 37.7121,
+      "eval_samples_per_second": 424.267,
+      "eval_steps_per_second": 6.629,
       "step": 200
     },
     {
       "epoch": 0.4,
-      "eval_loss": 0.21885745227336884,
-      "eval_runtime": 37.9737,
-      "eval_samples_per_second": 421.344,
-      "eval_steps_per_second": 6.583,
+      "eval_loss": 0.0523061603307724,
+      "eval_runtime": 37.8838,
+      "eval_samples_per_second": 422.344,
+      "eval_steps_per_second": 6.599,
       "step": 400
     },
     {
       "epoch": 0.5,
-      "grad_norm": 205354.109375,
-      "learning_rate": 4.877641290737884e-05,
-      "loss": 0.3926,
+      "grad_norm": 72862.21875,
+      "learning_rate": 4.9692208514878444e-05,
+      "loss": 0.1425,
       "step": 500
     },
     {
       "epoch": 0.6,
-      "eval_loss": 1.307199478149414,
-      "eval_runtime": 37.7639,
-      "eval_samples_per_second": 423.685,
-      "eval_steps_per_second": 6.62,
+      "eval_loss": 1.0542395114898682,
+      "eval_runtime": 37.8634,
+      "eval_samples_per_second": 422.571,
+      "eval_steps_per_second": 6.603,
       "step": 600
     },
     {
       "epoch": 0.8,
-      "eval_loss": 0.062136366963386536,
-      "eval_runtime": 37.7282,
-      "eval_samples_per_second": 424.086,
-      "eval_steps_per_second": 6.626,
+      "eval_loss": 1.0459295511245728,
+      "eval_runtime": 37.8174,
+      "eval_samples_per_second": 423.086,
+      "eval_steps_per_second": 6.611,
       "step": 800
     },
     {
       "epoch": 1.0,
-      "grad_norm": 0.6423090696334839,
-      "learning_rate": 4.522542485937369e-05,
-      "loss": 0.9884,
+      "grad_norm": 4.641770839691162,
+      "learning_rate": 4.877641290737884e-05,
+      "loss": 0.9177,
       "step": 1000
     },
     {
       "epoch": 1.0,
-      "eval_loss": 0.031570661813020706,
-      "eval_runtime": 37.8801,
-      "eval_samples_per_second": 422.385,
-      "eval_steps_per_second": 6.6,
+      "eval_loss": 0.3677258789539337,
+      "eval_runtime": 37.6734,
+      "eval_samples_per_second": 424.703,
+      "eval_steps_per_second": 6.636,
       "step": 1000
     },
     {
       "epoch": 1.2,
-      "eval_loss": 0.02908959612250328,
-      "eval_runtime": 37.7451,
-      "eval_samples_per_second": 423.896,
-      "eval_steps_per_second": 6.623,
+      "eval_loss": 0.02963975816965103,
+      "eval_runtime": 37.6097,
+      "eval_samples_per_second": 425.422,
+      "eval_steps_per_second": 6.647,
       "step": 1200
     },
     {
       "epoch": 1.4,
-      "eval_loss": 0.028545573353767395,
-      "eval_runtime": 37.8679,
-      "eval_samples_per_second": 422.522,
-      "eval_steps_per_second": 6.602,
+      "eval_loss": 0.028412258252501488,
+      "eval_runtime": 37.7795,
+      "eval_samples_per_second": 423.51,
+      "eval_steps_per_second": 6.617,
       "step": 1400
     },
     {
       "epoch": 1.5,
-      "grad_norm": 0.2492751181125641,
-      "learning_rate": 3.969463130731183e-05,
-      "loss": 0.0322,
+      "grad_norm": 0.17703795433044434,
+      "learning_rate": 4.72751631047092e-05,
+      "loss": 0.0421,
       "step": 1500
     },
     {
       "epoch": 1.6,
-      "eval_loss": 0.027546165511012077,
-      "eval_runtime": 37.9026,
-      "eval_samples_per_second": 422.135,
-      "eval_steps_per_second": 6.596,
+      "eval_loss": 0.02752041630446911,
+      "eval_runtime": 37.7317,
+      "eval_samples_per_second": 424.046,
+      "eval_steps_per_second": 6.626,
       "step": 1600
     },
     {
       "epoch": 1.8,
-      "eval_loss": 0.027202017605304718,
-      "eval_runtime": 37.8488,
-      "eval_samples_per_second": 422.735,
-      "eval_steps_per_second": 6.605,
+      "eval_loss": 0.027357231825590134,
+      "eval_runtime": 37.9192,
+      "eval_samples_per_second": 421.95,
+      "eval_steps_per_second": 6.593,
       "step": 1800
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.06460036337375641,
-      "learning_rate": 3.272542485937369e-05,
-      "loss": 0.0297,
+      "grad_norm": 0.14987020194530487,
+      "learning_rate": 4.522542485937369e-05,
+      "loss": 0.029,
       "step": 2000
     },
     {
       "epoch": 2.0,
-      "eval_loss": 0.026708217337727547,
-      "eval_runtime": 37.6675,
-      "eval_samples_per_second": 424.77,
-      "eval_steps_per_second": 6.637,
+      "eval_loss": 0.026410279795527458,
+      "eval_runtime": 37.883,
+      "eval_samples_per_second": 422.353,
+      "eval_steps_per_second": 6.599,
       "step": 2000
     },
     {
       "epoch": 2.2,
-      "eval_loss": 0.026396282017230988,
-      "eval_runtime": 37.8265,
-      "eval_samples_per_second": 422.984,
-      "eval_steps_per_second": 6.609,
+      "eval_loss": 0.025838496163487434,
+      "eval_runtime": 37.6759,
+      "eval_samples_per_second": 424.674,
+      "eval_steps_per_second": 6.636,
       "step": 2200
     },
     {
       "epoch": 2.4,
-      "eval_loss": 0.026144716888666153,
-      "eval_runtime": 37.693,
-      "eval_samples_per_second": 424.482,
-      "eval_steps_per_second": 6.633,
+      "eval_loss": 0.025557253509759903,
+      "eval_runtime": 37.6691,
+      "eval_samples_per_second": 424.751,
+      "eval_steps_per_second": 6.637,
       "step": 2400
     },
     {
       "epoch": 2.5,
-      "grad_norm": 0.17030462622642517,
-      "learning_rate": 2.5e-05,
-      "loss": 0.028,
+      "grad_norm": 0.21350397169589996,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 0.0276,
       "step": 2500
     },
     {
       "epoch": 2.6,
-      "eval_loss": 0.02573326788842678,
-      "eval_runtime": 37.6502,
-      "eval_samples_per_second": 424.964,
-      "eval_steps_per_second": 6.64,
+      "eval_loss": 0.025391312316060066,
+      "eval_runtime": 37.9185,
+      "eval_samples_per_second": 421.958,
+      "eval_steps_per_second": 6.593,
       "step": 2600
     },
     {
       "epoch": 2.8,
-      "eval_loss": 0.025387177243828773,
-      "eval_runtime": 37.6666,
-      "eval_samples_per_second": 424.779,
-      "eval_steps_per_second": 6.637,
+      "eval_loss": 0.025234265252947807,
+      "eval_runtime": 37.6569,
+      "eval_samples_per_second": 424.889,
+      "eval_steps_per_second": 6.639,
       "step": 2800
     },
     {
       "epoch": 3.0,
-      "grad_norm": 0.13623261451721191,
-      "learning_rate": 1.7274575140626318e-05,
-      "loss": 0.0268,
+      "grad_norm": 0.09507149457931519,
+      "learning_rate": 3.969463130731183e-05,
+      "loss": 0.0265,
       "step": 3000
     },
     {
       "epoch": 3.0,
-      "eval_loss": 0.02545003592967987,
-      "eval_runtime": 37.6642,
-      "eval_samples_per_second": 424.806,
-      "eval_steps_per_second": 6.638,
+      "eval_loss": 0.025119660422205925,
+      "eval_runtime": 37.7071,
+      "eval_samples_per_second": 424.323,
+      "eval_steps_per_second": 6.63,
       "step": 3000
     },
     {
       "epoch": 3.2,
-      "eval_loss": 0.02509310096502304,
-      "eval_runtime": 37.675,
-      "eval_samples_per_second": 424.685,
-      "eval_steps_per_second": 6.636,
+      "eval_loss": 0.02474472112953663,
+      "eval_runtime": 37.8472,
+      "eval_samples_per_second": 422.753,
+      "eval_steps_per_second": 6.606,
       "step": 3200
     },
     {
       "epoch": 3.4,
-      "eval_loss": 0.025246502831578255,
-      "eval_runtime": 37.7858,
-      "eval_samples_per_second": 423.439,
-      "eval_steps_per_second": 6.616,
+      "eval_loss": 0.02474530041217804,
+      "eval_runtime": 37.964,
+      "eval_samples_per_second": 421.452,
+      "eval_steps_per_second": 6.585,
       "step": 3400
     },
     {
       "epoch": 3.5,
-      "grad_norm": 0.12589260935783386,
-      "learning_rate": 1.0305368692688174e-05,
-      "loss": 0.0263,
+      "grad_norm": 0.21493718028068542,
+      "learning_rate": 3.634976249348867e-05,
+      "loss": 0.0256,
       "step": 3500
     },
     {
       "epoch": 3.6,
-      "eval_loss": 0.0253463052213192,
-      "eval_runtime": 37.879,
-      "eval_samples_per_second": 422.398,
-      "eval_steps_per_second": 6.6,
+      "eval_loss": 0.024641884490847588,
+      "eval_runtime": 37.7028,
+      "eval_samples_per_second": 424.372,
+      "eval_steps_per_second": 6.631,
       "step": 3600
     },
     {
       "epoch": 3.8,
-      "eval_loss": 0.024967506527900696,
-      "eval_runtime": 82.198,
-      "eval_samples_per_second": 194.652,
-      "eval_steps_per_second": 3.041,
+      "eval_loss": 0.02516881749033928,
+      "eval_runtime": 37.6697,
+      "eval_samples_per_second": 424.745,
+      "eval_steps_per_second": 6.637,
       "step": 3800
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.10492005199193954,
-      "learning_rate": 4.7745751406263165e-06,
-      "loss": 0.0264,
+      "grad_norm": 0.11537094414234161,
+      "learning_rate": 3.272542485937369e-05,
+      "loss": 0.0262,
       "step": 4000
     },
     {
       "epoch": 4.0,
-      "eval_loss": 0.02466404065489769,
-      "eval_runtime": 82.3457,
-      "eval_samples_per_second": 194.303,
-      "eval_steps_per_second": 3.036,
+      "eval_loss": 0.024907398968935013,
+      "eval_runtime": 37.8134,
+      "eval_samples_per_second": 423.13,
+      "eval_steps_per_second": 6.611,
       "step": 4000
-    },
-    {
-      "epoch": 4.2,
-      "eval_loss": 0.02478621155023575,
-      "eval_runtime": 76.6082,
-      "eval_samples_per_second": 208.855,
-      "eval_steps_per_second": 3.263,
-      "step": 4200
-    },
-    {
-      "epoch": 4.4,
-      "eval_loss": 0.024716826155781746,
-      "eval_runtime": 86.8007,
-      "eval_samples_per_second": 184.33,
-      "eval_steps_per_second": 2.88,
-      "step": 4400
-    },
-    {
-      "epoch": 4.5,
-      "grad_norm": 0.1692284792661667,
-      "learning_rate": 1.2235870926211619e-06,
-      "loss": 0.026,
-      "step": 4500
-    },
-    {
-      "epoch": 4.6,
-      "eval_loss": 0.024610303342342377,
-      "eval_runtime": 82.2894,
-      "eval_samples_per_second": 194.436,
-      "eval_steps_per_second": 3.038,
-      "step": 4600
-    },
-    {
-      "epoch": 4.8,
-      "eval_loss": 0.024647582322359085,
-      "eval_runtime": 83.9599,
-      "eval_samples_per_second": 190.567,
-      "eval_steps_per_second": 2.978,
-      "step": 4800
-    },
-    {
-      "epoch": 5.0,
-      "grad_norm": 0.16388757526874542,
-      "learning_rate": 0.0,
-      "loss": 0.0258,
-      "step": 5000
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 0.02461400255560875,
-      "eval_runtime": 73.1695,
-      "eval_samples_per_second": 218.67,
-      "eval_steps_per_second": 3.417,
-      "step": 5000
     }
   ],
   "logging_steps": 500,
-  "max_steps": 5000,
+  "max_steps": 10000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 10,
   "save_steps": 1000,
-  "total_flos": 2.090336256e+16,
+  "total_flos": 1.6722690048e+16,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1f0fcad3e74386ffcf1917836bd60c8d585b4f88d87c9181c320cc9fefea09b
+oid sha256:ac099e73b43ec4fb0ff55120cb27fe4c7253b82e028a96ded709dbe7962ba7d3
 size 4984