Model save

README.md

@@ -27,14 +27,14 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/mzyin-university-of-florida/huggingface/runs/o18nogg0) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/mzyin-university-of-florida/huggingface/runs/vkhqux38) 
 
 
 This model was trained with SFT.
 
 ### Framework versions
 
-- TRL: 0.14.0
+- TRL: 0.15.2
 - Transformers: 4.48.3
 - Pytorch: 2.2.2+cu121
 - Datasets: 3.2.0

adapter_config.json

@@ -23,8 +23,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj"
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76d465a76c22fc53b903b8d9d2e1adabb5b72c7fe6f306a205847f0774c7c107
+oid sha256:794e0236da548e1f70ce43e7de7d10780ffe1eb7d297bcc86cdae95d5875da4f
 size 545743328

all_results.json

@@ -5,10 +5,10 @@
     "eval_samples": 142,
     "eval_samples_per_second": 30.075,
     "eval_steps_per_second": 5.013,
-    "total_flos": 72813143654400.0,
-    "train_loss": 3.3806732257207233,
-    "train_runtime": 123.5801,
+    "total_flos": 184078172160000.0,
+    "train_loss": 3.2382628750801086,
+    "train_runtime": 305.3611,
     "train_samples": 568,
-    "train_samples_per_second": 14.404,
-    "train_steps_per_second": 1.942
+    "train_samples_per_second": 14.737,
+    "train_steps_per_second": 1.965
 }

train_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 20.0,
-    "total_flos": 72813143654400.0,
-    "train_loss": 3.3806732257207233,
-    "train_runtime": 123.5801,
+    "total_flos": 184078172160000.0,
+    "train_loss": 3.2382628750801086,
+    "train_runtime": 305.3611,
     "train_samples": 568,
-    "train_samples_per_second": 14.404,
-    "train_steps_per_second": 1.942
+    "train_samples_per_second": 14.737,
+    "train_steps_per_second": 1.965
 }

trainer_state.json

@@ -1,530 +1,1445 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 20.0,
+  "epoch": 50.0,
   "eval_steps": 500,
-  "global_step": 240,
+  "global_step": 600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.08333333333333333,
-      "grad_norm": 1.9178808308871647,
-      "learning_rate": 8.333333333333333e-07,
-      "loss": 3.4904,
+      "grad_norm": 0.5385836071499915,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 3.5061,
+      "mean_token_accuracy": 0.3923509418964386,
       "step": 1
     },
     {
       "epoch": 0.4166666666666667,
-      "grad_norm": 1.5508324121618542,
-      "learning_rate": 4.166666666666667e-06,
-      "loss": 3.613,
+      "grad_norm": 0.5988682460559892,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 3.444,
+      "mean_token_accuracy": 0.40139296650886536,
       "step": 5
     },
     {
       "epoch": 0.8333333333333334,
-      "grad_norm": 3.970063835806634,
-      "learning_rate": 8.333333333333334e-06,
-      "loss": 3.5534,
+      "grad_norm": 0.7225027190447023,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 3.469,
+      "mean_token_accuracy": 0.40673128962516786,
       "step": 10
     },
     {
       "epoch": 1.0,
-      "eval_loss": 3.635176420211792,
-      "eval_runtime": 0.3923,
-      "eval_samples_per_second": 45.887,
-      "eval_steps_per_second": 7.648,
+      "eval_loss": 3.5662965774536133,
+      "eval_mean_token_accuracy": 0.4117915093898773,
+      "eval_runtime": 0.5523,
+      "eval_samples_per_second": 34.404,
+      "eval_steps_per_second": 5.432,
       "step": 12
     },
     {
       "epoch": 1.25,
-      "grad_norm": 1.7466258721475718,
-      "learning_rate": 1.25e-05,
-      "loss": 3.4791,
+      "grad_norm": 0.5723484530658475,
+      "learning_rate": 5e-06,
+      "loss": 3.4415,
+      "mean_token_accuracy": 0.40831703941027325,
       "step": 15
     },
     {
       "epoch": 1.6666666666666665,
-      "grad_norm": 1.1618839918387402,
-      "learning_rate": 1.6666666666666667e-05,
-      "loss": 3.5451,
+      "grad_norm": 0.7162755459145884,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 3.4582,
+      "mean_token_accuracy": 0.4077712595462799,
       "step": 20
     },
     {
       "epoch": 2.0,
-      "eval_loss": 3.5881924629211426,
-      "eval_runtime": 0.3977,
-      "eval_samples_per_second": 45.263,
-      "eval_steps_per_second": 7.544,
+      "eval_loss": 3.5607330799102783,
+      "eval_mean_token_accuracy": 0.4010107176644461,
+      "eval_runtime": 0.4408,
+      "eval_samples_per_second": 43.103,
+      "eval_steps_per_second": 6.806,
       "step": 24
     },
     {
       "epoch": 2.0833333333333335,
-      "grad_norm": 1.0276735175803224,
-      "learning_rate": 1.9998942319271076e-05,
-      "loss": 3.4481,
+      "grad_norm": 0.5761807107644672,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 3.4905,
+      "mean_token_accuracy": 0.40334799885749817,
       "step": 25
     },
     {
       "epoch": 2.5,
-      "grad_norm": 1.0269510028877782,
-      "learning_rate": 1.9961946980917457e-05,
-      "loss": 3.5143,
+      "grad_norm": 0.5868666256020767,
+      "learning_rate": 1e-05,
+      "loss": 3.3955,
+      "mean_token_accuracy": 0.4089615702629089,
       "step": 30
     },
     {
       "epoch": 2.9166666666666665,
-      "grad_norm": 0.8284784581301078,
-      "learning_rate": 1.9872291131173743e-05,
-      "loss": 3.4974,
+      "grad_norm": 0.7608176868125189,
+      "learning_rate": 1.1666666666666668e-05,
+      "loss": 3.4528,
+      "mean_token_accuracy": 0.40273705720901487,
       "step": 35
     },
     {
       "epoch": 3.0,
-      "eval_loss": 3.5418310165405273,
-      "eval_runtime": 0.3965,
-      "eval_samples_per_second": 45.397,
-      "eval_steps_per_second": 7.566,
+      "eval_loss": 3.5521068572998047,
+      "eval_mean_token_accuracy": 0.41008396446704865,
+      "eval_runtime": 0.4318,
+      "eval_samples_per_second": 43.998,
+      "eval_steps_per_second": 6.947,
       "step": 36
     },
     {
       "epoch": 3.3333333333333335,
-      "grad_norm": 0.8328561838416785,
-      "learning_rate": 1.973044870579824e-05,
-      "loss": 3.4338,
+      "grad_norm": 0.6096220864944857,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 3.5249,
+      "mean_token_accuracy": 0.4006356969475746,
       "step": 40
     },
     {
       "epoch": 3.75,
-      "grad_norm": 1.0109709585778517,
-      "learning_rate": 1.953716950748227e-05,
-      "loss": 3.4939,
+      "grad_norm": 0.5375302227210489,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 3.4592,
+      "mean_token_accuracy": 0.40808895230293274,
       "step": 45
     },
     {
       "epoch": 4.0,
-      "eval_loss": 3.5025124549865723,
-      "eval_runtime": 0.3969,
-      "eval_samples_per_second": 45.35,
-      "eval_steps_per_second": 7.558,
+      "eval_loss": 3.5394978523254395,
+      "eval_mean_token_accuracy": 0.41112928589185077,
+      "eval_runtime": 0.4311,
+      "eval_samples_per_second": 44.074,
+      "eval_steps_per_second": 6.959,
       "step": 48
     },
     {
       "epoch": 4.166666666666667,
-      "grad_norm": 0.7400417939368705,
-      "learning_rate": 1.9293475242268224e-05,
-      "loss": 3.3865,
+      "grad_norm": 0.5093292635141339,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 3.4,
+      "mean_token_accuracy": 0.404203325510025,
       "step": 50
     },
     {
       "epoch": 4.583333333333333,
-      "grad_norm": 0.7420166459361851,
-      "learning_rate": 1.900065411864121e-05,
-      "loss": 3.452,
+      "grad_norm": 0.7243587198171676,
+      "learning_rate": 1.8333333333333333e-05,
+      "loss": 3.4412,
+      "mean_token_accuracy": 0.4058379828929901,
       "step": 55
     },
     {
       "epoch": 5.0,
-      "grad_norm": 1.864771582064087,
-      "learning_rate": 1.866025403784439e-05,
-      "loss": 3.3788,
+      "grad_norm": 0.8522717487007759,
+      "learning_rate": 2e-05,
+      "loss": 3.3885,
+      "mean_token_accuracy": 0.4081867039203644,
       "step": 60
     },
     {
       "epoch": 5.0,
-      "eval_loss": 3.4763269424438477,
-      "eval_runtime": 0.3912,
-      "eval_samples_per_second": 46.015,
-      "eval_steps_per_second": 7.669,
+      "eval_loss": 3.523003101348877,
+      "eval_mean_token_accuracy": 0.41269586483637494,
+      "eval_runtime": 0.4315,
+      "eval_samples_per_second": 44.032,
+      "eval_steps_per_second": 6.952,
       "step": 60
     },
     {
       "epoch": 5.416666666666667,
-      "grad_norm": 0.7205181158730114,
-      "learning_rate": 1.8274074411415104e-05,
-      "loss": 3.3805,
+      "grad_norm": 0.5197308524638413,
+      "learning_rate": 1.9995769500822007e-05,
+      "loss": 3.4699,
+      "mean_token_accuracy": 0.40173509120941164,
       "step": 65
     },
     {
       "epoch": 5.833333333333333,
-      "grad_norm": 0.5903690958172121,
-      "learning_rate": 1.784415664919576e-05,
-      "loss": 3.4453,
+      "grad_norm": 0.6040667002909094,
+      "learning_rate": 1.9983081582712684e-05,
+      "loss": 3.341,
+      "mean_token_accuracy": 0.4136533796787262,
       "step": 70
     },
     {
       "epoch": 6.0,
-      "eval_loss": 3.45723557472229,
-      "eval_runtime": 0.4034,
-      "eval_samples_per_second": 44.624,
-      "eval_steps_per_second": 7.437,
+      "eval_loss": 3.5047221183776855,
+      "eval_mean_token_accuracy": 0.4145672619342804,
+      "eval_runtime": 0.4362,
+      "eval_samples_per_second": 43.562,
+      "eval_steps_per_second": 6.878,
       "step": 72
     },
     {
       "epoch": 6.25,
-      "grad_norm": 0.5437262967095119,
-      "learning_rate": 1.737277336810124e-05,
-      "loss": 3.4045,
+      "grad_norm": 0.48368556562954307,
+      "learning_rate": 1.9961946980917457e-05,
+      "loss": 3.4062,
+      "mean_token_accuracy": 0.4024519423643748,
       "step": 75
     },
     {
       "epoch": 6.666666666666667,
-      "grad_norm": 0.5235425065858881,
-      "learning_rate": 1.686241637868734e-05,
-      "loss": 3.3306,
+      "grad_norm": 0.624860448459548,
+      "learning_rate": 1.9932383577419432e-05,
+      "loss": 3.3999,
+      "mean_token_accuracy": 0.41031280159950256,
       "step": 80
     },
     {
       "epoch": 7.0,
-      "eval_loss": 3.441802501678467,
-      "eval_runtime": 0.4108,
-      "eval_samples_per_second": 43.819,
-      "eval_steps_per_second": 7.303,
+      "eval_loss": 3.4868686199188232,
+      "eval_mean_token_accuracy": 0.41539154733930317,
+      "eval_runtime": 0.4392,
+      "eval_samples_per_second": 43.261,
+      "eval_steps_per_second": 6.831,
       "step": 84
     },
     {
       "epoch": 7.083333333333333,
-      "grad_norm": 0.6513847176886255,
-      "learning_rate": 1.6315783513024977e-05,
-      "loss": 3.5057,
+      "grad_norm": 0.5271197118444322,
+      "learning_rate": 1.9894416385809444e-05,
+      "loss": 3.3464,
+      "mean_token_accuracy": 0.395772248506546,
       "step": 85
     },
     {
       "epoch": 7.5,
-      "grad_norm": 0.5155086538623761,
-      "learning_rate": 1.573576436351046e-05,
-      "loss": 3.378,
+      "grad_norm": 0.6058343803263453,
+      "learning_rate": 1.9848077530122083e-05,
+      "loss": 3.332,
+      "mean_token_accuracy": 0.4168132901191711,
       "step": 90
     },
     {
       "epoch": 7.916666666666667,
-      "grad_norm": 0.5143935752297035,
-      "learning_rate": 1.5125425007998653e-05,
-      "loss": 3.345,
+      "grad_norm": 0.5498361997261547,
+      "learning_rate": 1.9793406217655516e-05,
+      "loss": 3.3951,
+      "mean_token_accuracy": 0.40805121660232546,
       "step": 95
     },
     {
       "epoch": 8.0,
-      "eval_loss": 3.428921937942505,
-      "eval_runtime": 0.3894,
-      "eval_samples_per_second": 46.225,
-      "eval_steps_per_second": 7.704,
+      "eval_loss": 3.4701385498046875,
+      "eval_mean_token_accuracy": 0.417536236345768,
+      "eval_runtime": 0.4284,
+      "eval_samples_per_second": 44.348,
+      "eval_steps_per_second": 7.002,
       "step": 96
     },
     {
       "epoch": 8.333333333333334,
-      "grad_norm": 0.5652349397643913,
-      "learning_rate": 1.4487991802004625e-05,
-      "loss": 3.5027,
+      "grad_norm": 0.49021333765538,
+      "learning_rate": 1.973044870579824e-05,
+      "loss": 3.316,
+      "mean_token_accuracy": 0.41507820039987564,
       "step": 100
     },
     {
       "epoch": 8.75,
-      "grad_norm": 0.5647161437519873,
-      "learning_rate": 1.3826834323650899e-05,
-      "loss": 3.3973,
+      "grad_norm": 0.5876084010033806,
+      "learning_rate": 1.9659258262890683e-05,
+      "loss": 3.4082,
+      "mean_token_accuracy": 0.4088488757610321,
       "step": 105
     },
     {
       "epoch": 9.0,
-      "eval_loss": 3.418201446533203,
-      "eval_runtime": 0.4033,
-      "eval_samples_per_second": 44.627,
-      "eval_steps_per_second": 7.438,
+      "eval_loss": 3.4551661014556885,
+      "eval_mean_token_accuracy": 0.41253359615802765,
+      "eval_runtime": 0.4291,
+      "eval_samples_per_second": 44.28,
+      "eval_steps_per_second": 6.992,
       "step": 108
     },
     {
       "epoch": 9.166666666666666,
-      "grad_norm": 0.5444861959273378,
-      "learning_rate": 1.3145447561516138e-05,
-      "loss": 3.2417,
+      "grad_norm": 0.5075915346441139,
+      "learning_rate": 1.957989512315489e-05,
+      "loss": 3.3659,
+      "mean_token_accuracy": 0.4220624566078186,
       "step": 110
     },
     {
       "epoch": 9.583333333333334,
-      "grad_norm": 0.5831760497922609,
-      "learning_rate": 1.2447433439543239e-05,
-      "loss": 3.369,
+      "grad_norm": 0.47536348642658954,
+      "learning_rate": 1.949242643573034e-05,
+      "loss": 3.3739,
+      "mean_token_accuracy": 0.40786901116371155,
       "step": 115
     },
     {
       "epoch": 10.0,
-      "grad_norm": 1.019730158221383,
-      "learning_rate": 1.1736481776669307e-05,
-      "loss": 3.2892,
+      "grad_norm": 0.8249517268614861,
+      "learning_rate": 1.9396926207859085e-05,
+      "loss": 3.3766,
+      "mean_token_accuracy": 0.4057429075241089,
       "step": 120
     },
     {
       "epoch": 10.0,
-      "eval_loss": 3.4091298580169678,
-      "eval_runtime": 0.4621,
-      "eval_samples_per_second": 38.949,
-      "eval_steps_per_second": 6.492,
+      "eval_loss": 3.440880298614502,
+      "eval_mean_token_accuracy": 0.4163530071576436,
+      "eval_runtime": 0.4465,
+      "eval_samples_per_second": 42.557,
+      "eval_steps_per_second": 6.72,
       "step": 120
     },
     {
       "epoch": 10.416666666666666,
-      "grad_norm": 0.5472889256976743,
-      "learning_rate": 1.101635078182802e-05,
-      "loss": 3.3538,
+      "grad_norm": 0.5247440587444925,
+      "learning_rate": 1.9293475242268224e-05,
+      "loss": 3.3593,
+      "mean_token_accuracy": 0.40792441964149473,
       "step": 125
     },
     {
       "epoch": 10.833333333333334,
-      "grad_norm": 0.585130945106004,
-      "learning_rate": 1.0290847187431115e-05,
-      "loss": 3.3218,
+      "grad_norm": 0.7084057790066337,
+      "learning_rate": 1.9182161068802742e-05,
+      "loss": 3.3108,
+      "mean_token_accuracy": 0.4163245320320129,
       "step": 130
     },
     {
       "epoch": 11.0,
-      "eval_loss": 3.401224374771118,
-      "eval_runtime": 0.3959,
-      "eval_samples_per_second": 45.464,
-      "eval_steps_per_second": 7.577,
+      "eval_loss": 3.426577091217041,
+      "eval_mean_token_accuracy": 0.41616382598876955,
+      "eval_runtime": 0.4272,
+      "eval_samples_per_second": 44.48,
+      "eval_steps_per_second": 7.023,
       "step": 132
     },
     {
       "epoch": 11.25,
-      "grad_norm": 0.49009027634210744,
-      "learning_rate": 9.563806126346643e-06,
-      "loss": 3.3543,
+      "grad_norm": 0.5224822341776986,
+      "learning_rate": 1.9063077870366504e-05,
+      "loss": 3.2782,
+      "mean_token_accuracy": 0.4194770356019338,
       "step": 135
     },
     {
       "epoch": 11.666666666666666,
-      "grad_norm": 0.5409056672592139,
-      "learning_rate": 8.839070858747697e-06,
-      "loss": 3.3185,
+      "grad_norm": 0.45694283255218365,
+      "learning_rate": 1.8936326403234125e-05,
+      "loss": 3.3102,
+      "mean_token_accuracy": 0.41273156404495237,
       "step": 140
     },
     {
       "epoch": 12.0,
-      "eval_loss": 3.394521951675415,
-      "eval_runtime": 0.4021,
-      "eval_samples_per_second": 44.762,
-      "eval_steps_per_second": 7.46,
+      "eval_loss": 3.4135658740997314,
+      "eval_mean_token_accuracy": 0.41462882501738413,
+      "eval_runtime": 0.4547,
+      "eval_samples_per_second": 41.783,
+      "eval_steps_per_second": 6.597,
       "step": 144
     },
     {
       "epoch": 12.083333333333334,
-      "grad_norm": 0.5879385039357604,
-      "learning_rate": 8.120472455998882e-06,
-      "loss": 3.4697,
+      "grad_norm": 0.4475822239307412,
+      "learning_rate": 1.880201391180111e-05,
+      "loss": 3.37,
+      "mean_token_accuracy": 0.40713587403297424,
       "step": 145
     },
     {
       "epoch": 12.5,
-      "grad_norm": 0.5109001687995364,
-      "learning_rate": 7.411809548974792e-06,
-      "loss": 3.293,
+      "grad_norm": 0.5801727962051494,
+      "learning_rate": 1.866025403784439e-05,
+      "loss": 3.3554,
+      "mean_token_accuracy": 0.41182795763015745,
       "step": 150
     },
     {
       "epoch": 12.916666666666666,
-      "grad_norm": 0.5648980578882977,
-      "learning_rate": 6.716828247864391e-06,
-      "loss": 3.3237,
+      "grad_norm": 0.5874670469007709,
+      "learning_rate": 1.8511166724369997e-05,
+      "loss": 3.2614,
+      "mean_token_accuracy": 0.4187495827674866,
       "step": 155
     },
     {
       "epoch": 13.0,
-      "eval_loss": 3.3890793323516846,
-      "eval_runtime": 0.3967,
-      "eval_samples_per_second": 45.374,
-      "eval_steps_per_second": 7.562,
+      "eval_loss": 3.4014406204223633,
+      "eval_mean_token_accuracy": 0.4147283807396889,
+      "eval_runtime": 0.4294,
+      "eval_samples_per_second": 44.244,
+      "eval_steps_per_second": 6.986,
       "step": 156
     },
     {
       "epoch": 13.333333333333334,
-      "grad_norm": 0.5055559054562139,
-      "learning_rate": 6.039202339608432e-06,
-      "loss": 3.271,
+      "grad_norm": 0.550319404785184,
+      "learning_rate": 1.8354878114129368e-05,
+      "loss": 3.2805,
+      "mean_token_accuracy": 0.414131224155426,
       "step": 160
     },
     {
       "epoch": 13.75,
-      "grad_norm": 0.5732954250338061,
-      "learning_rate": 5.382513867649663e-06,
-      "loss": 3.3964,
+      "grad_norm": 0.5904370361609361,
+      "learning_rate": 1.819152044288992e-05,
+      "loss": 3.2972,
+      "mean_token_accuracy": 0.41746036410331727,
       "step": 165
     },
     {
       "epoch": 14.0,
-      "eval_loss": 3.3846511840820312,
-      "eval_runtime": 0.395,
-      "eval_samples_per_second": 45.566,
-      "eval_steps_per_second": 7.594,
+      "eval_loss": 3.3907692432403564,
+      "eval_mean_token_accuracy": 0.4185921351114909,
+      "eval_runtime": 0.428,
+      "eval_samples_per_second": 44.396,
+      "eval_steps_per_second": 7.01,
       "step": 168
     },
     {
       "epoch": 14.166666666666666,
-      "grad_norm": 0.5692530517966546,
-      "learning_rate": 4.7502341966544e-06,
-      "loss": 3.1924,
+      "grad_norm": 0.5164762014150277,
+      "learning_rate": 1.802123192755044e-05,
+      "loss": 3.2903,
+      "mean_token_accuracy": 0.41526149213314056,
       "step": 170
     },
     {
       "epoch": 14.583333333333334,
-      "grad_norm": 0.5478826916587504,
-      "learning_rate": 4.1457056623005954e-06,
-      "loss": 3.3224,
+      "grad_norm": 0.6449998942915702,
+      "learning_rate": 1.784415664919576e-05,
+      "loss": 3.2937,
+      "mean_token_accuracy": 0.4124247670173645,
       "step": 175
     },
     {
       "epoch": 15.0,
-      "grad_norm": 1.3475215612190545,
-      "learning_rate": 3.5721239031346067e-06,
-      "loss": 3.3293,
+      "grad_norm": 0.9850442727409189,
+      "learning_rate": 1.766044443118978e-05,
+      "loss": 3.2699,
+      "mean_token_accuracy": 0.41933039426803587,
       "step": 180
     },
     {
       "epoch": 15.0,
-      "eval_loss": 3.3815081119537354,
-      "eval_runtime": 0.3939,
-      "eval_samples_per_second": 45.698,
-      "eval_steps_per_second": 7.616,
+      "eval_loss": 3.380833625793457,
+      "eval_mean_token_accuracy": 0.4198370774586995,
+      "eval_runtime": 0.4534,
+      "eval_samples_per_second": 41.906,
+      "eval_steps_per_second": 6.617,
       "step": 180
     },
     {
       "epoch": 15.416666666666666,
-      "grad_norm": 0.49662476648041026,
-      "learning_rate": 3.032520967893453e-06,
-      "loss": 3.321,
+      "grad_norm": 0.5357760809508904,
+      "learning_rate": 1.7470250712409963e-05,
+      "loss": 3.2893,
+      "mean_token_accuracy": 0.4150129497051239,
       "step": 185
     },
     {
       "epoch": 15.833333333333334,
-      "grad_norm": 0.5516226107095812,
-      "learning_rate": 2.529749287590042e-06,
-      "loss": 3.2824,
+      "grad_norm": 0.6312781940606058,
+      "learning_rate": 1.7273736415730488e-05,
+      "loss": 3.2636,
+      "mean_token_accuracy": 0.41713098883628846,
       "step": 190
     },
     {
       "epoch": 16.0,
-      "eval_loss": 3.3792712688446045,
-      "eval_runtime": 0.3974,
-      "eval_samples_per_second": 45.291,
-      "eval_steps_per_second": 7.548,
+      "eval_loss": 3.3715903759002686,
+      "eval_mean_token_accuracy": 0.41947720646858216,
+      "eval_runtime": 0.5943,
+      "eval_samples_per_second": 31.971,
+      "eval_steps_per_second": 5.048,
       "step": 192
     },
     {
       "epoch": 16.25,
-      "grad_norm": 0.5663384865119242,
-      "learning_rate": 2.0664665970876496e-06,
-      "loss": 3.3657,
+      "grad_norm": 0.5229064222364138,
+      "learning_rate": 1.7071067811865477e-05,
+      "loss": 3.2393,
+      "mean_token_accuracy": 0.41088302930196124,
       "step": 195
     },
     {
       "epoch": 16.666666666666668,
-      "grad_norm": 0.6014416895030616,
-      "learning_rate": 1.6451218858706374e-06,
-      "loss": 3.3232,
+      "grad_norm": 0.517899162096867,
+      "learning_rate": 1.686241637868734e-05,
+      "loss": 3.2543,
+      "mean_token_accuracy": 0.4211387991905212,
       "step": 200
     },
     {
       "epoch": 17.0,
-      "eval_loss": 3.377903461456299,
-      "eval_runtime": 0.405,
-      "eval_samples_per_second": 44.444,
-      "eval_steps_per_second": 7.407,
+      "eval_loss": 3.3629448413848877,
+      "eval_mean_token_accuracy": 0.41943295512880596,
+      "eval_runtime": 0.4287,
+      "eval_samples_per_second": 44.317,
+      "eval_steps_per_second": 6.997,
       "step": 204
     },
     {
       "epoch": 17.083333333333332,
-      "grad_norm": 0.4906231418660385,
-      "learning_rate": 1.2679424522780426e-06,
-      "loss": 3.3418,
+      "grad_norm": 0.5400119094778694,
+      "learning_rate": 1.6647958656139377e-05,
+      "loss": 3.2249,
+      "mean_token_accuracy": 0.4193548262119293,
       "step": 205
     },
     {
       "epoch": 17.5,
-      "grad_norm": 0.5991421773344411,
-      "learning_rate": 9.369221296335007e-07,
-      "loss": 3.3385,
+      "grad_norm": 0.488184108091038,
+      "learning_rate": 1.6427876096865394e-05,
+      "loss": 3.1743,
+      "mean_token_accuracy": 0.42307103872299195,
       "step": 210
     },
     {
       "epoch": 17.916666666666668,
-      "grad_norm": 0.572738899557872,
-      "learning_rate": 6.538107465101162e-07,
-      "loss": 3.2725,
+      "grad_norm": 0.6484010807393733,
+      "learning_rate": 1.6202354912682602e-05,
+      "loss": 3.3001,
+      "mean_token_accuracy": 0.41410067677497864,
       "step": 215
     },
     {
       "epoch": 18.0,
-      "eval_loss": 3.3772037029266357,
-      "eval_runtime": 0.4012,
-      "eval_samples_per_second": 44.861,
-      "eval_steps_per_second": 7.477,
+      "eval_loss": 3.354940176010132,
+      "eval_mean_token_accuracy": 0.41730327904224396,
+      "eval_runtime": 0.431,
+      "eval_samples_per_second": 44.089,
+      "eval_steps_per_second": 6.961,
       "step": 216
     },
     {
       "epoch": 18.333333333333332,
-      "grad_norm": 0.6238290992567466,
-      "learning_rate": 4.2010487684511105e-07,
-      "loss": 3.4302,
+      "grad_norm": 0.5674421464466002,
+      "learning_rate": 1.5971585917027864e-05,
+      "loss": 3.2051,
+      "mean_token_accuracy": 0.4223179370164871,
       "step": 220
     },
     {
       "epoch": 18.75,
-      "grad_norm": 0.5767194960270463,
-      "learning_rate": 2.370399288006664e-07,
-      "loss": 3.2808,
+      "grad_norm": 0.5903205133298044,
+      "learning_rate": 1.573576436351046e-05,
+      "loss": 3.3103,
+      "mean_token_accuracy": 0.4130250930786133,
       "step": 225
     },
     {
       "epoch": 19.0,
-      "eval_loss": 3.3768930435180664,
-      "eval_runtime": 0.3991,
-      "eval_samples_per_second": 45.102,
-      "eval_steps_per_second": 7.517,
+      "eval_loss": 3.347010850906372,
+      "eval_mean_token_accuracy": 0.4260312815507253,
+      "eval_runtime": 0.4239,
+      "eval_samples_per_second": 44.822,
+      "eval_steps_per_second": 7.077,
       "step": 228
     },
     {
       "epoch": 19.166666666666668,
-      "grad_norm": 0.5557398310185774,
-      "learning_rate": 1.055836141905553e-07,
-      "loss": 3.3581,
+      "grad_norm": 0.6190609888103829,
+      "learning_rate": 1.5495089780708062e-05,
+      "loss": 3.2368,
+      "mean_token_accuracy": 0.4261363744735718,
       "step": 230
     },
     {
       "epoch": 19.583333333333332,
-      "grad_norm": 0.5389585071398526,
-      "learning_rate": 2.643083299427751e-08,
-      "loss": 3.3357,
+      "grad_norm": 0.6450479176223197,
+      "learning_rate": 1.5249765803345602e-05,
+      "loss": 3.2506,
+      "mean_token_accuracy": 0.41713098287582395,
       "step": 235
     },
     {
       "epoch": 20.0,
-      "grad_norm": 1.3350882978965304,
-      "learning_rate": 0.0,
-      "loss": 3.3157,
+      "grad_norm": 0.9163763629667666,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 3.1941,
+      "mean_token_accuracy": 0.41965908408164976,
       "step": 240
     },
     {
       "epoch": 20.0,
-      "eval_loss": 3.3768398761749268,
-      "eval_runtime": 0.3925,
-      "eval_samples_per_second": 45.863,
-      "eval_steps_per_second": 7.644,
+      "eval_loss": 3.339346170425415,
+      "eval_mean_token_accuracy": 0.42223334312438965,
+      "eval_runtime": 0.5247,
+      "eval_samples_per_second": 36.208,
+      "eval_steps_per_second": 5.717,
       "step": 240
     },
     {
-      "epoch": 20.0,
-      "step": 240,
-      "total_flos": 72813143654400.0,
-      "train_loss": 3.3806732257207233,
-      "train_runtime": 123.5801,
-      "train_samples_per_second": 14.404,
-      "train_steps_per_second": 1.942
+      "epoch": 20.416666666666668,
+      "grad_norm": 0.6468751066404622,
+      "learning_rate": 1.4746003697476406e-05,
+      "loss": 3.2192,
+      "mean_token_accuracy": 0.4208211123943329,
+      "step": 245
+    },
+    {
+      "epoch": 20.833333333333332,
+      "grad_norm": 0.6683774245030316,
+      "learning_rate": 1.4487991802004625e-05,
+      "loss": 3.2221,
+      "mean_token_accuracy": 0.4196500062942505,
+      "step": 250
+    },
+    {
+      "epoch": 21.0,
+      "eval_loss": 3.331696033477783,
+      "eval_mean_token_accuracy": 0.4289926946163177,
+      "eval_runtime": 0.4271,
+      "eval_samples_per_second": 44.486,
+      "eval_steps_per_second": 7.024,
+      "step": 252
+    },
+    {
+      "epoch": 21.25,
+      "grad_norm": 0.5643106555296927,
+      "learning_rate": 1.4226182617406996e-05,
+      "loss": 3.2169,
+      "mean_token_accuracy": 0.41858096917470294,
+      "step": 255
+    },
+    {
+      "epoch": 21.666666666666668,
+      "grad_norm": 0.5977762587235801,
+      "learning_rate": 1.396079766039157e-05,
+      "loss": 3.2027,
+      "mean_token_accuracy": 0.4227761447429657,
+      "step": 260
+    },
+    {
+      "epoch": 22.0,
+      "eval_loss": 3.324859142303467,
+      "eval_mean_token_accuracy": 0.41909594195229666,
+      "eval_runtime": 0.4259,
+      "eval_samples_per_second": 44.615,
+      "eval_steps_per_second": 7.044,
+      "step": 264
+    },
+    {
+      "epoch": 22.083333333333332,
+      "grad_norm": 0.6213218140554986,
+      "learning_rate": 1.3692061473126845e-05,
+      "loss": 3.2305,
+      "mean_token_accuracy": 0.4119012653827667,
+      "step": 265
+    },
+    {
+      "epoch": 22.5,
+      "grad_norm": 0.6275730529051374,
+      "learning_rate": 1.342020143325669e-05,
+      "loss": 3.1752,
+      "mean_token_accuracy": 0.42231183052062987,
+      "step": 270
+    },
+    {
+      "epoch": 22.916666666666668,
+      "grad_norm": 0.5390571854463481,
+      "learning_rate": 1.3145447561516138e-05,
+      "loss": 3.2192,
+      "mean_token_accuracy": 0.4236361861228943,
+      "step": 275
+    },
+    {
+      "epoch": 23.0,
+      "eval_loss": 3.31854248046875,
+      "eval_mean_token_accuracy": 0.4187733605504036,
+      "eval_runtime": 0.4331,
+      "eval_samples_per_second": 43.866,
+      "eval_steps_per_second": 6.926,
+      "step": 276
+    },
+    {
+      "epoch": 23.333333333333332,
+      "grad_norm": 0.54125472321236,
+      "learning_rate": 1.2868032327110904e-05,
+      "loss": 3.2537,
+      "mean_token_accuracy": 0.41950757056474686,
+      "step": 280
+    },
+    {
+      "epoch": 23.75,
+      "grad_norm": 0.5722463822439342,
+      "learning_rate": 1.2588190451025209e-05,
+      "loss": 3.1885,
+      "mean_token_accuracy": 0.4205010116100311,
+      "step": 285
+    },
+    {
+      "epoch": 24.0,
+      "eval_loss": 3.3127944469451904,
+      "eval_mean_token_accuracy": 0.422769491871198,
+      "eval_runtime": 0.4219,
+      "eval_samples_per_second": 45.038,
+      "eval_steps_per_second": 7.111,
+      "step": 288
+    },
+    {
+      "epoch": 24.166666666666668,
+      "grad_norm": 0.679559216877849,
+      "learning_rate": 1.2306158707424402e-05,
+      "loss": 3.2156,
+      "mean_token_accuracy": 0.42124877870082855,
+      "step": 290
+    },
+    {
+      "epoch": 24.583333333333332,
+      "grad_norm": 0.5610092093025837,
+      "learning_rate": 1.2022175723320382e-05,
+      "loss": 3.1649,
+      "mean_token_accuracy": 0.42517107129096987,
+      "step": 295
+    },
+    {
+      "epoch": 25.0,
+      "grad_norm": 1.426121211640697,
+      "learning_rate": 1.1736481776669307e-05,
+      "loss": 3.2052,
+      "mean_token_accuracy": 0.42243062853813174,
+      "step": 300
+    },
+    {
+      "epoch": 25.0,
+      "eval_loss": 3.307049512863159,
+      "eval_mean_token_accuracy": 0.4235536555449168,
+      "eval_runtime": 0.4166,
+      "eval_samples_per_second": 45.602,
+      "eval_steps_per_second": 7.2,
+      "step": 300
+    },
+    {
+      "epoch": 25.416666666666668,
+      "grad_norm": 0.5665773108951218,
+      "learning_rate": 1.1449318593072468e-05,
+      "loss": 3.1675,
+      "mean_token_accuracy": 0.4225806474685669,
+      "step": 305
+    },
+    {
+      "epoch": 25.833333333333332,
+      "grad_norm": 0.6226920179957871,
+      "learning_rate": 1.1160929141252303e-05,
+      "loss": 3.192,
+      "mean_token_accuracy": 0.42431179285049436,
+      "step": 310
+    },
+    {
+      "epoch": 26.0,
+      "eval_loss": 3.3022124767303467,
+      "eval_mean_token_accuracy": 0.4199638843536377,
+      "eval_runtime": 0.4381,
+      "eval_samples_per_second": 43.371,
+      "eval_steps_per_second": 6.848,
+      "step": 312
+    },
+    {
+      "epoch": 26.25,
+      "grad_norm": 0.6087103966453086,
+      "learning_rate": 1.0871557427476585e-05,
+      "loss": 3.2605,
+      "mean_token_accuracy": 0.417011300722758,
+      "step": 315
+    },
+    {
+      "epoch": 26.666666666666668,
+      "grad_norm": 0.5698423662256255,
+      "learning_rate": 1.0581448289104759e-05,
+      "loss": 3.1541,
+      "mean_token_accuracy": 0.4270772337913513,
+      "step": 320
+    },
+    {
+      "epoch": 27.0,
+      "eval_loss": 3.297753095626831,
+      "eval_mean_token_accuracy": 0.42913591861724854,
+      "eval_runtime": 0.4251,
+      "eval_samples_per_second": 44.69,
+      "eval_steps_per_second": 7.056,
+      "step": 324
+    },
+    {
+      "epoch": 27.083333333333332,
+      "grad_norm": 0.6813246252702757,
+      "learning_rate": 1.0290847187431115e-05,
+      "loss": 3.1759,
+      "mean_token_accuracy": 0.419721394777298,
+      "step": 325
+    },
+    {
+      "epoch": 27.5,
+      "grad_norm": 0.5500989419947021,
+      "learning_rate": 1e-05,
+      "loss": 3.1638,
+      "mean_token_accuracy": 0.4259925663471222,
+      "step": 330
+    },
+    {
+      "epoch": 27.916666666666668,
+      "grad_norm": 0.5981580816134079,
+      "learning_rate": 9.709152812568886e-06,
+      "loss": 3.174,
+      "mean_token_accuracy": 0.4233870983123779,
+      "step": 335
+    },
+    {
+      "epoch": 28.0,
+      "eval_loss": 3.293494462966919,
+      "eval_mean_token_accuracy": 0.4238861948251724,
+      "eval_runtime": 0.4411,
+      "eval_samples_per_second": 43.07,
+      "eval_steps_per_second": 6.8,
+      "step": 336
+    },
+    {
+      "epoch": 28.333333333333332,
+      "grad_norm": 0.5930311679061705,
+      "learning_rate": 9.418551710895243e-06,
+      "loss": 3.1868,
+      "mean_token_accuracy": 0.4231025353074074,
+      "step": 340
+    },
+    {
+      "epoch": 28.75,
+      "grad_norm": 0.5754287350322016,
+      "learning_rate": 9.128442572523418e-06,
+      "loss": 3.1808,
+      "mean_token_accuracy": 0.4236803472042084,
+      "step": 345
+    },
+    {
+      "epoch": 29.0,
+      "eval_loss": 3.2898471355438232,
+      "eval_mean_token_accuracy": 0.4265649865070979,
+      "eval_runtime": 0.4268,
+      "eval_samples_per_second": 44.518,
+      "eval_steps_per_second": 7.029,
+      "step": 348
+    },
+    {
+      "epoch": 29.166666666666668,
+      "grad_norm": 0.604278267795617,
+      "learning_rate": 8.839070858747697e-06,
+      "loss": 3.2018,
+      "mean_token_accuracy": 0.41849951446056366,
+      "step": 350
+    },
+    {
+      "epoch": 29.583333333333332,
+      "grad_norm": 0.6525176961848282,
+      "learning_rate": 8.550681406927534e-06,
+      "loss": 3.1601,
+      "mean_token_accuracy": 0.42639296054840087,
+      "step": 355
+    },
+    {
+      "epoch": 30.0,
+      "grad_norm": 1.3214766985272957,
+      "learning_rate": 8.263518223330698e-06,
+      "loss": 3.2003,
+      "mean_token_accuracy": 0.4250500977039337,
+      "step": 360
+    },
+    {
+      "epoch": 30.0,
+      "eval_loss": 3.2868618965148926,
+      "eval_mean_token_accuracy": 0.42555957039197284,
+      "eval_runtime": 0.4245,
+      "eval_samples_per_second": 44.754,
+      "eval_steps_per_second": 7.066,
+      "step": 360
+    },
+    {
+      "epoch": 30.416666666666668,
+      "grad_norm": 0.5951289198915272,
+      "learning_rate": 7.977824276679623e-06,
+      "loss": 3.1575,
+      "mean_token_accuracy": 0.4253473997116089,
+      "step": 365
+    },
+    {
+      "epoch": 30.833333333333332,
+      "grad_norm": 0.6178093593706783,
+      "learning_rate": 7.6938412925756e-06,
+      "loss": 3.1917,
+      "mean_token_accuracy": 0.4213587462902069,
+      "step": 370
+    },
+    {
+      "epoch": 31.0,
+      "eval_loss": 3.2837605476379395,
+      "eval_mean_token_accuracy": 0.4344413161277771,
+      "eval_runtime": 0.437,
+      "eval_samples_per_second": 43.477,
+      "eval_steps_per_second": 6.865,
+      "step": 372
+    },
+    {
+      "epoch": 31.25,
+      "grad_norm": 0.7052320977702828,
+      "learning_rate": 7.411809548974792e-06,
+      "loss": 3.1153,
+      "mean_token_accuracy": 0.4197881321112315,
+      "step": 375
+    },
+    {
+      "epoch": 31.666666666666668,
+      "grad_norm": 0.5285913317420398,
+      "learning_rate": 7.131967672889101e-06,
+      "loss": 3.1552,
+      "mean_token_accuracy": 0.42627077698707583,
+      "step": 380
+    },
+    {
+      "epoch": 32.0,
+      "eval_loss": 3.2812047004699707,
+      "eval_mean_token_accuracy": 0.42681941390037537,
+      "eval_runtime": 0.449,
+      "eval_samples_per_second": 42.32,
+      "eval_steps_per_second": 6.682,
+      "step": 384
+    },
+    {
+      "epoch": 32.083333333333336,
+      "grad_norm": 0.5722095071093364,
+      "learning_rate": 6.854552438483866e-06,
+      "loss": 3.1829,
+      "mean_token_accuracy": 0.4199657738208771,
+      "step": 385
+    },
+    {
+      "epoch": 32.5,
+      "grad_norm": 0.7633904658788278,
+      "learning_rate": 6.579798566743314e-06,
+      "loss": 3.1684,
+      "mean_token_accuracy": 0.422702831029892,
+      "step": 390
+    },
+    {
+      "epoch": 32.916666666666664,
+      "grad_norm": 0.7552346932336533,
+      "learning_rate": 6.3079385268731575e-06,
+      "loss": 3.1554,
+      "mean_token_accuracy": 0.42810051441192626,
+      "step": 395
+    },
+    {
+      "epoch": 33.0,
+      "eval_loss": 3.279100179672241,
+      "eval_mean_token_accuracy": 0.4279579147696495,
+      "eval_runtime": 0.4241,
+      "eval_samples_per_second": 44.798,
+      "eval_steps_per_second": 7.073,
+      "step": 396
+    },
+    {
+      "epoch": 33.333333333333336,
+      "grad_norm": 0.6276366843863107,
+      "learning_rate": 6.039202339608432e-06,
+      "loss": 3.1644,
+      "mean_token_accuracy": 0.4224095791578293,
+      "step": 400
+    },
+    {
+      "epoch": 33.75,
+      "grad_norm": 0.6661697390297467,
+      "learning_rate": 5.773817382593008e-06,
+      "loss": 3.1175,
+      "mean_token_accuracy": 0.43118279576301577,
+      "step": 405
+    },
+    {
+      "epoch": 34.0,
+      "eval_loss": 3.2770273685455322,
+      "eval_mean_token_accuracy": 0.41977598269780475,
+      "eval_runtime": 0.4259,
+      "eval_samples_per_second": 44.612,
+      "eval_steps_per_second": 7.044,
+      "step": 408
+    },
+    {
+      "epoch": 34.166666666666664,
+      "grad_norm": 0.6396786596872565,
+      "learning_rate": 5.512008197995379e-06,
+      "loss": 3.2052,
+      "mean_token_accuracy": 0.42155425250530243,
+      "step": 410
+    },
+    {
+      "epoch": 34.583333333333336,
+      "grad_norm": 0.7372526034823661,
+      "learning_rate": 5.253996302523596e-06,
+      "loss": 3.1537,
+      "mean_token_accuracy": 0.42629474997520445,
+      "step": 415
+    },
+    {
+      "epoch": 35.0,
+      "grad_norm": 1.3769536069183301,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": 3.152,
+      "mean_token_accuracy": 0.42785924077034,
+      "step": 420
+    },
+    {
+      "epoch": 35.0,
+      "eval_loss": 3.275233507156372,
+      "eval_mean_token_accuracy": 0.4261179069677989,
+      "eval_runtime": 0.4346,
+      "eval_samples_per_second": 43.72,
+      "eval_steps_per_second": 6.903,
+      "step": 420
+    },
+    {
+      "epoch": 35.416666666666664,
+      "grad_norm": 0.607101085973905,
+      "learning_rate": 4.7502341966544e-06,
+      "loss": 3.157,
+      "mean_token_accuracy": 0.4252443790435791,
+      "step": 425
+    },
+    {
+      "epoch": 35.833333333333336,
+      "grad_norm": 0.6902217168837974,
+      "learning_rate": 4.504910219291941e-06,
+      "loss": 3.2004,
+      "mean_token_accuracy": 0.42346973419189454,
+      "step": 430
+    },
+    {
+      "epoch": 36.0,
+      "eval_loss": 3.2738254070281982,
+      "eval_mean_token_accuracy": 0.43027973771095274,
+      "eval_runtime": 0.4432,
+      "eval_samples_per_second": 42.866,
+      "eval_steps_per_second": 6.768,
+      "step": 432
+    },
+    {
+      "epoch": 36.25,
+      "grad_norm": 0.6833076608186851,
+      "learning_rate": 4.264235636489542e-06,
+      "loss": 3.0659,
+      "mean_token_accuracy": 0.426602840423584,
+      "step": 435
+    },
+    {
+      "epoch": 36.666666666666664,
+      "grad_norm": 0.5918683696670577,
+      "learning_rate": 4.028414082972141e-06,
+      "loss": 3.1312,
+      "mean_token_accuracy": 0.42512218952178954,
+      "step": 440
+    },
+    {
+      "epoch": 37.0,
+      "eval_loss": 3.2725744247436523,
+      "eval_mean_token_accuracy": 0.42391158853258404,
+      "eval_runtime": 0.4318,
+      "eval_samples_per_second": 44.001,
+      "eval_steps_per_second": 6.947,
+      "step": 444
+    },
+    {
+      "epoch": 37.083333333333336,
+      "grad_norm": 0.6190943636017486,
+      "learning_rate": 3.797645087317401e-06,
+      "loss": 3.1971,
+      "mean_token_accuracy": 0.4199657738208771,
+      "step": 445
+    },
+    {
+      "epoch": 37.5,
+      "grad_norm": 0.7187996022384058,
+      "learning_rate": 3.5721239031346067e-06,
+      "loss": 3.1919,
+      "mean_token_accuracy": 0.42737048864364624,
+      "step": 450
+    },
+    {
+      "epoch": 37.916666666666664,
+      "grad_norm": 0.5959621609292731,
+      "learning_rate": 3.3520413438606215e-06,
+      "loss": 3.122,
+      "mean_token_accuracy": 0.425152450799942,
+      "step": 455
+    },
+    {
+      "epoch": 38.0,
+      "eval_loss": 3.2714850902557373,
+      "eval_mean_token_accuracy": 0.42657437175512314,
+      "eval_runtime": 0.4313,
+      "eval_samples_per_second": 44.056,
+      "eval_steps_per_second": 6.956,
+      "step": 456
+    },
+    {
+      "epoch": 38.333333333333336,
+      "grad_norm": 0.5510804324691496,
+      "learning_rate": 3.1375836213126653e-06,
+      "loss": 3.1652,
+      "mean_token_accuracy": 0.4300769865512848,
+      "step": 460
+    },
+    {
+      "epoch": 38.75,
+      "grad_norm": 0.7109787627150623,
+      "learning_rate": 2.9289321881345257e-06,
+      "loss": 3.0988,
+      "mean_token_accuracy": 0.4267013967037201,
+      "step": 465
+    },
+    {
+      "epoch": 39.0,
+      "eval_loss": 3.270362615585327,
+      "eval_mean_token_accuracy": 0.4215492556492488,
+      "eval_runtime": 0.4303,
+      "eval_samples_per_second": 44.155,
+      "eval_steps_per_second": 6.972,
+      "step": 468
+    },
+    {
+      "epoch": 39.166666666666664,
+      "grad_norm": 0.774771996806958,
+      "learning_rate": 2.726263584269513e-06,
+      "loss": 3.1793,
+      "mean_token_accuracy": 0.4258919805288315,
+      "step": 470
+    },
+    {
+      "epoch": 39.583333333333336,
+      "grad_norm": 0.6131163287949375,
+      "learning_rate": 2.529749287590042e-06,
+      "loss": 3.1375,
+      "mean_token_accuracy": 0.42778592705726626,
+      "step": 475
+    },
+    {
+      "epoch": 40.0,
+      "grad_norm": 1.2153887550210514,
+      "learning_rate": 2.339555568810221e-06,
+      "loss": 3.145,
+      "mean_token_accuracy": 0.42437560558319093,
+      "step": 480
+    },
+    {
+      "epoch": 40.0,
+      "eval_loss": 3.2696785926818848,
+      "eval_mean_token_accuracy": 0.4261993666489919,
+      "eval_runtime": 0.429,
+      "eval_samples_per_second": 44.291,
+      "eval_steps_per_second": 6.993,
+      "step": 480
+    },
+    {
+      "epoch": 40.416666666666664,
+      "grad_norm": 0.6720557667088608,
+      "learning_rate": 2.155843350804243e-06,
+      "loss": 3.1445,
+      "mean_token_accuracy": 0.42810567617416384,
+      "step": 485
+    },
+    {
+      "epoch": 40.833333333333336,
+      "grad_norm": 0.6250108698940896,
+      "learning_rate": 1.9787680724495617e-06,
+      "loss": 3.1776,
+      "mean_token_accuracy": 0.42465786933898925,
+      "step": 490
+    },
+    {
+      "epoch": 41.0,
+      "eval_loss": 3.2690887451171875,
+      "eval_mean_token_accuracy": 0.4265896141529083,
+      "eval_runtime": 0.4278,
+      "eval_samples_per_second": 44.411,
+      "eval_steps_per_second": 7.012,
+      "step": 492
+    },
+    {
+      "epoch": 41.25,
+      "grad_norm": 0.5760551828751769,
+      "learning_rate": 1.808479557110081e-06,
+      "loss": 3.0955,
+      "mean_token_accuracy": 0.42819322148958844,
+      "step": 495
+    },
+    {
+      "epoch": 41.666666666666664,
+      "grad_norm": 0.6656339982260218,
+      "learning_rate": 1.6451218858706374e-06,
+      "loss": 3.1127,
+      "mean_token_accuracy": 0.4301280200481415,
+      "step": 500
+    },
+    {
+      "epoch": 42.0,
+      "eval_loss": 3.268610954284668,
+      "eval_mean_token_accuracy": 0.42469198788915363,
+      "eval_runtime": 0.4334,
+      "eval_samples_per_second": 43.84,
+      "eval_steps_per_second": 6.922,
+      "step": 504
+    },
+    {
+      "epoch": 42.083333333333336,
+      "grad_norm": 0.7475978754963217,
+      "learning_rate": 1.4888332756300027e-06,
+      "loss": 3.1808,
+      "mean_token_accuracy": 0.423631489276886,
+      "step": 505
+    },
+    {
+      "epoch": 42.5,
+      "grad_norm": 0.6501237190023742,
+      "learning_rate": 1.339745962155613e-06,
+      "loss": 3.1527,
+      "mean_token_accuracy": 0.4245112419128418,
+      "step": 510
+    },
+    {
+      "epoch": 42.916666666666664,
+      "grad_norm": 0.62569750833363,
+      "learning_rate": 1.1979860881988903e-06,
+      "loss": 3.1386,
+      "mean_token_accuracy": 0.428892993927002,
+      "step": 515
+    },
+    {
+      "epoch": 43.0,
+      "eval_loss": 3.268239974975586,
+      "eval_mean_token_accuracy": 0.42221880704164505,
+      "eval_runtime": 0.43,
+      "eval_samples_per_second": 44.182,
+      "eval_steps_per_second": 6.976,
+      "step": 516
+    },
+    {
+      "epoch": 43.333333333333336,
+      "grad_norm": 0.554243721179707,
+      "learning_rate": 1.0636735967658785e-06,
+      "loss": 3.1772,
+      "mean_token_accuracy": 0.42421653121709824,
+      "step": 520
+    },
+    {
+      "epoch": 43.75,
+      "grad_norm": 0.5721327568599583,
+      "learning_rate": 9.369221296335007e-07,
+      "loss": 3.1272,
+      "mean_token_accuracy": 0.4287878811359406,
+      "step": 525
+    },
+    {
+      "epoch": 44.0,
+      "eval_loss": 3.267944812774658,
+      "eval_mean_token_accuracy": 0.4253371407588323,
+      "eval_runtime": 0.4335,
+      "eval_samples_per_second": 43.832,
+      "eval_steps_per_second": 6.921,
+      "step": 528
+    },
+    {
+      "epoch": 44.166666666666664,
+      "grad_norm": 0.7430429454372077,
+      "learning_rate": 8.178389311972612e-07,
+      "loss": 3.1615,
+      "mean_token_accuracy": 0.4282746911048889,
+      "step": 530
+    },
+    {
+      "epoch": 44.583333333333336,
+      "grad_norm": 0.7042749042309677,
+      "learning_rate": 7.065247577317747e-07,
+      "loss": 3.1264,
+      "mean_token_accuracy": 0.42416911125183104,
+      "step": 535
+    },
+    {
+      "epoch": 45.0,
+      "grad_norm": 1.2711783589705017,
+      "learning_rate": 6.030737921409169e-07,
+      "loss": 3.1803,
+      "mean_token_accuracy": 0.42598907351493837,
+      "step": 540
+    },
+    {
+      "epoch": 45.0,
+      "eval_loss": 3.2677195072174072,
+      "eval_mean_token_accuracy": 0.42655404408772785,
+      "eval_runtime": 0.4208,
+      "eval_samples_per_second": 45.15,
+      "eval_steps_per_second": 7.129,
+      "step": 540
+    },
+    {
+      "epoch": 45.416666666666664,
+      "grad_norm": 0.5778581622717375,
+      "learning_rate": 5.075735642696611e-07,
+      "loss": 3.1695,
+      "mean_token_accuracy": 0.42243402600288393,
+      "step": 545
+    },
+    {
+      "epoch": 45.833333333333336,
+      "grad_norm": 0.6459085305134198,
+      "learning_rate": 4.2010487684511105e-07,
+      "loss": 3.1143,
+      "mean_token_accuracy": 0.4312965631484985,
+      "step": 550
+    },
+    {
+      "epoch": 46.0,
+      "eval_loss": 3.267563819885254,
+      "eval_mean_token_accuracy": 0.42787768244743346,
+      "eval_runtime": 0.4291,
+      "eval_samples_per_second": 44.276,
+      "eval_steps_per_second": 6.991,
+      "step": 552
+    },
+    {
+      "epoch": 46.25,
+      "grad_norm": 0.6444220526276241,
+      "learning_rate": 3.4074173710931804e-07,
+      "loss": 3.1236,
+      "mean_token_accuracy": 0.4287661810715993,
+      "step": 555
+    },
+    {
+      "epoch": 46.666666666666664,
+      "grad_norm": 0.610353826593815,
+      "learning_rate": 2.6955129420176193e-07,
+      "loss": 3.1322,
+      "mean_token_accuracy": 0.42734603881835936,
+      "step": 560
+    },
+    {
+      "epoch": 47.0,
+      "eval_loss": 3.2675440311431885,
+      "eval_mean_token_accuracy": 0.4225522152015141,
+      "eval_runtime": 0.4219,
+      "eval_samples_per_second": 45.036,
+      "eval_steps_per_second": 7.111,
+      "step": 564
+    },
+    {
+      "epoch": 47.083333333333336,
+      "grad_norm": 0.5779765132480464,
+      "learning_rate": 2.0659378234448524e-07,
+      "loss": 3.2209,
+      "mean_token_accuracy": 0.41336753964424133,
+      "step": 565
+    },
+    {
+      "epoch": 47.5,
+      "grad_norm": 0.6848925907095541,
+      "learning_rate": 1.519224698779198e-07,
+      "loss": 3.142,
+      "mean_token_accuracy": 0.42521933317184446,
+      "step": 570
+    },
+    {
+      "epoch": 47.916666666666664,
+      "grad_norm": 0.6437140874495073,
+      "learning_rate": 1.055836141905553e-07,
+      "loss": 3.1318,
+      "mean_token_accuracy": 0.42976540327072144,
+      "step": 575
+    },
+    {
+      "epoch": 48.0,
+      "eval_loss": 3.267465114593506,
+      "eval_mean_token_accuracy": 0.42656926065683365,
+      "eval_runtime": 0.4269,
+      "eval_samples_per_second": 44.507,
+      "eval_steps_per_second": 7.027,
+      "step": 576
+    },
+    {
+      "epoch": 48.333333333333336,
+      "grad_norm": 0.7317817126043084,
+      "learning_rate": 6.761642258056977e-08,
+      "loss": 3.1684,
+      "mean_token_accuracy": 0.43004219233989716,
+      "step": 580
+    },
+    {
+      "epoch": 48.75,
+      "grad_norm": 0.7021129395579033,
+      "learning_rate": 3.805301908254455e-08,
+      "loss": 3.1435,
+      "mean_token_accuracy": 0.4232893466949463,
+      "step": 585
+    },
+    {
+      "epoch": 49.0,
+      "eval_loss": 3.267409563064575,
+      "eval_mean_token_accuracy": 0.42759764691193897,
+      "eval_runtime": 0.4339,
+      "eval_samples_per_second": 43.788,
+      "eval_steps_per_second": 6.914,
+      "step": 588
+    },
+    {
+      "epoch": 49.166666666666664,
+      "grad_norm": 0.6452805374112162,
+      "learning_rate": 1.6918417287318245e-08,
+      "loss": 3.1535,
+      "mean_token_accuracy": 0.43218475580215454,
+      "step": 590
+    },
+    {
+      "epoch": 49.583333333333336,
+      "grad_norm": 0.6574552722398359,
+      "learning_rate": 4.230499177994007e-09,
+      "loss": 3.1514,
+      "mean_token_accuracy": 0.42807917594909667,
+      "step": 595
+    },
+    {
+      "epoch": 50.0,
+      "grad_norm": 1.3863477885366933,
+      "learning_rate": 0.0,
+      "loss": 3.1944,
+      "mean_token_accuracy": 0.4211083292961121,
+      "step": 600
+    },
+    {
+      "epoch": 50.0,
+      "eval_loss": 3.2674479484558105,
+      "eval_mean_token_accuracy": 0.42677465081214905,
+      "eval_runtime": 0.4251,
+      "eval_samples_per_second": 44.694,
+      "eval_steps_per_second": 7.057,
+      "step": 600
+    },
+    {
+      "epoch": 50.0,
+      "step": 600,
+      "total_flos": 184078172160000.0,
+      "train_loss": 3.2382628750801086,
+      "train_runtime": 305.3611,
+      "train_samples_per_second": 14.737,
+      "train_steps_per_second": 1.965
     }
   ],
   "logging_steps": 5,
-  "max_steps": 240,
+  "max_steps": 600,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 20,
+  "num_train_epochs": 50,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -538,7 +1453,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 72813143654400.0,
+  "total_flos": 184078172160000.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc6572bf040f8f4b9d9b41da1a82e4a0cef80f4d2912446b1ea044029db09e64
+oid sha256:d9c239b6680cc72b75187bd68708a7121b3a7a5ffa31e0eeeaf4c66e0742042b
 size 7544