End of training

README.md

@@ -2,11 +2,23 @@
 library_name: transformers
 tags:
 - generated_from_trainer
+datasets:
+- kanishka/babylm2-rewritten-clean-spacy
 metrics:
 - accuracy
 model-index:
 - name: opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3
-  results: []
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: kanishka/babylm2-rewritten-clean-spacy
+      type: kanishka/babylm2-rewritten-clean-spacy
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.47877642614021604
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -14,7 +26,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the kanishka/babylm2-rewritten-clean-spacy dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.6820
 - Accuracy: 0.4788

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 19.998892396300604,
-    "eval_accuracy": 0.47865689612852264,
-    "eval_loss": 2.6880221366882324,
-    "eval_runtime": 120.6441,
-    "eval_samples": 60791,
-    "eval_samples_per_second": 503.887,
-    "eval_steps_per_second": 7.874,
-    "perplexity": 14.702567474871172,
-    "total_flos": 1.50966240067584e+18,
-    "train_loss": 2.8030111154835997,
-    "train_runtime": 43302.4979,
-    "train_samples": 577799,
-    "train_samples_per_second": 266.866,
-    "train_steps_per_second": 1.042
+    "epoch": 20.0,
+    "eval_accuracy": 0.47877642614021604,
+    "eval_loss": 2.6820449829101562,
+    "eval_runtime": 71.9233,
+    "eval_samples": 60701,
+    "eval_samples_per_second": 843.969,
+    "eval_steps_per_second": 13.195,
+    "perplexity": 14.614950080315884,
+    "total_flos": 1.50902942072832e+18,
+    "train_loss": 2.805498681169875,
+    "train_runtime": 30524.4807,
+    "train_samples": 577526,
+    "train_samples_per_second": 378.402,
+    "train_steps_per_second": 1.478
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 19.998892396300604,
-    "eval_accuracy": 0.47865689612852264,
-    "eval_loss": 2.6880221366882324,
-    "eval_runtime": 120.6441,
-    "eval_samples": 60791,
-    "eval_samples_per_second": 503.887,
-    "eval_steps_per_second": 7.874,
-    "perplexity": 14.702567474871172
+    "epoch": 20.0,
+    "eval_accuracy": 0.47877642614021604,
+    "eval_loss": 2.6820449829101562,
+    "eval_runtime": 71.9233,
+    "eval_samples": 60701,
+    "eval_samples_per_second": 843.969,
+    "eval_steps_per_second": 13.195,
+    "perplexity": 14.614950080315884
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 19.998892396300604,
-    "total_flos": 1.50966240067584e+18,
-    "train_loss": 2.8030111154835997,
-    "train_runtime": 43302.4979,
-    "train_samples": 577799,
-    "train_samples_per_second": 266.866,
-    "train_steps_per_second": 1.042
+    "epoch": 20.0,
+    "total_flos": 1.50902942072832e+18,
+    "train_loss": 2.805498681169875,
+    "train_runtime": 30524.4807,
+    "train_samples": 577526,
+    "train_samples_per_second": 378.402,
+    "train_steps_per_second": 1.478
 }

trainer_state.json

@@ -1,520 +1,520 @@
 {
-  "best_metric": 2.6880221366882324,
-  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3/checkpoint-45140",
-  "epoch": 19.998892396300604,
+  "best_metric": 2.6820449829101562,
+  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3/checkpoint-45120",
+  "epoch": 20.0,
   "eval_steps": 500,
-  "global_step": 45140,
+  "global_step": 45120,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.4430414797585424,
-      "grad_norm": 0.4268507957458496,
+      "epoch": 0.4432624113475177,
+      "grad_norm": 0.469483345746994,
       "learning_rate": 3.125e-05,
-      "loss": 5.5906,
+      "loss": 5.5946,
       "step": 1000
     },
     {
-      "epoch": 0.8860829595170848,
-      "grad_norm": 0.5278797149658203,
+      "epoch": 0.8865248226950354,
+      "grad_norm": 0.6834925413131714,
       "learning_rate": 6.25e-05,
-      "loss": 4.0959,
+      "loss": 4.101,
       "step": 2000
     },
     {
-      "epoch": 0.9999446198150301,
-      "eval_accuracy": 0.3612965154478169,
-      "eval_loss": 3.8126308917999268,
-      "eval_runtime": 121.9395,
-      "eval_samples_per_second": 498.534,
-      "eval_steps_per_second": 7.791,
-      "step": 2257
+      "epoch": 1.0,
+      "eval_accuracy": 0.36108013855119486,
+      "eval_loss": 3.813789129257202,
+      "eval_runtime": 74.5004,
+      "eval_samples_per_second": 814.774,
+      "eval_steps_per_second": 12.738,
+      "step": 2256
     },
     {
-      "epoch": 1.3291244392756272,
-      "grad_norm": 0.5504615306854248,
+      "epoch": 1.3297872340425532,
+      "grad_norm": 0.5719049572944641,
       "learning_rate": 9.375e-05,
-      "loss": 3.6988,
+      "loss": 3.7004,
       "step": 3000
     },
     {
-      "epoch": 1.7721659190341694,
-      "grad_norm": 0.5281194448471069,
+      "epoch": 1.773049645390071,
+      "grad_norm": 0.5357337594032288,
       "learning_rate": 0.000125,
-      "loss": 3.4463,
+      "loss": 3.445,
       "step": 4000
     },
     {
-      "epoch": 1.9998892396300603,
-      "eval_accuracy": 0.4098802035001956,
-      "eval_loss": 3.297177791595459,
-      "eval_runtime": 122.3693,
-      "eval_samples_per_second": 496.783,
-      "eval_steps_per_second": 7.763,
-      "step": 4514
+      "epoch": 2.0,
+      "eval_accuracy": 0.4097865752122829,
+      "eval_loss": 3.29646635055542,
+      "eval_runtime": 74.2471,
+      "eval_samples_per_second": 817.554,
+      "eval_steps_per_second": 12.782,
+      "step": 4512
     },
     {
-      "epoch": 2.215207398792712,
-      "grad_norm": 0.491211473941803,
+      "epoch": 2.2163120567375887,
+      "grad_norm": 0.5515570640563965,
       "learning_rate": 0.00015625,
-      "loss": 3.2482,
+      "loss": 3.2498,
       "step": 5000
     },
     {
-      "epoch": 2.6582488785512544,
-      "grad_norm": 0.5076552033424377,
+      "epoch": 2.6595744680851063,
+      "grad_norm": 0.5060694813728333,
       "learning_rate": 0.0001875,
-      "loss": 3.1228,
+      "loss": 3.1279,
       "step": 6000
     },
     {
-      "epoch": 2.9998338594450904,
-      "eval_accuracy": 0.4315480135894729,
-      "eval_loss": 3.085052013397217,
-      "eval_runtime": 122.1579,
-      "eval_samples_per_second": 497.643,
-      "eval_steps_per_second": 7.777,
-      "step": 6771
+      "epoch": 3.0,
+      "eval_accuracy": 0.43082928827286177,
+      "eval_loss": 3.0860400199890137,
+      "eval_runtime": 74.3001,
+      "eval_samples_per_second": 816.97,
+      "eval_steps_per_second": 12.773,
+      "step": 6768
     },
     {
-      "epoch": 3.1012903583097966,
-      "grad_norm": 0.4416440725326538,
+      "epoch": 3.102836879432624,
+      "grad_norm": 0.4440550208091736,
       "learning_rate": 0.00021875,
-      "loss": 3.0284,
+      "loss": 3.0384,
       "step": 7000
     },
     {
-      "epoch": 3.5443318380683393,
-      "grad_norm": 0.37537524104118347,
+      "epoch": 3.546099290780142,
+      "grad_norm": 0.4073255956172943,
       "learning_rate": 0.00025,
-      "loss": 2.9593,
+      "loss": 2.9651,
       "step": 8000
     },
     {
-      "epoch": 3.9873733178268815,
-      "grad_norm": 0.39330974221229553,
+      "epoch": 3.9893617021276597,
+      "grad_norm": 0.3835934102535248,
       "learning_rate": 0.00028125000000000003,
-      "loss": 2.9166,
+      "loss": 2.9218,
       "step": 9000
     },
     {
-      "epoch": 3.9997784792601205,
-      "eval_accuracy": 0.44179488643346004,
-      "eval_loss": 2.980708599090576,
-      "eval_runtime": 122.9667,
-      "eval_samples_per_second": 494.37,
-      "eval_steps_per_second": 7.726,
-      "step": 9028
+      "epoch": 4.0,
+      "eval_accuracy": 0.4411697193992669,
+      "eval_loss": 2.9823455810546875,
+      "eval_runtime": 73.4266,
+      "eval_samples_per_second": 826.69,
+      "eval_steps_per_second": 12.924,
+      "step": 9024
     },
     {
-      "epoch": 4.430414797585424,
-      "grad_norm": 0.370112806558609,
+      "epoch": 4.432624113475177,
+      "grad_norm": 0.35862067341804504,
       "learning_rate": 0.0003125,
-      "loss": 2.8563,
+      "loss": 2.8625,
       "step": 10000
     },
     {
-      "epoch": 4.8734562773439665,
-      "grad_norm": 0.3323429524898529,
+      "epoch": 4.875886524822695,
+      "grad_norm": 0.33855435252189636,
       "learning_rate": 0.00034375,
-      "loss": 2.8402,
+      "loss": 2.8441,
       "step": 11000
     },
     {
-      "epoch": 4.999723099075151,
-      "eval_accuracy": 0.44756373573100505,
-      "eval_loss": 2.924882411956787,
-      "eval_runtime": 122.7484,
-      "eval_samples_per_second": 495.249,
-      "eval_steps_per_second": 7.739,
-      "step": 11285
+      "epoch": 5.0,
+      "eval_accuracy": 0.44725845198790215,
+      "eval_loss": 2.9202077388763428,
+      "eval_runtime": 73.099,
+      "eval_samples_per_second": 830.394,
+      "eval_steps_per_second": 12.982,
+      "step": 11280
     },
     {
-      "epoch": 5.316497757102509,
-      "grad_norm": 0.31359294056892395,
-      "learning_rate": 0.00037496875000000003,
-      "loss": 2.7944,
+      "epoch": 5.319148936170213,
+      "grad_norm": 0.3236384987831116,
+      "learning_rate": 0.000375,
+      "loss": 2.7995,
       "step": 12000
     },
     {
-      "epoch": 5.759539236861051,
-      "grad_norm": 0.29497697949409485,
-      "learning_rate": 0.00040621875,
-      "loss": 2.7832,
+      "epoch": 5.76241134751773,
+      "grad_norm": 0.3051661252975464,
+      "learning_rate": 0.00040625000000000004,
+      "loss": 2.7865,
       "step": 13000
     },
     {
-      "epoch": 5.999667718890181,
-      "eval_accuracy": 0.45211594466544164,
-      "eval_loss": 2.8850650787353516,
-      "eval_runtime": 122.4548,
-      "eval_samples_per_second": 496.436,
-      "eval_steps_per_second": 7.758,
-      "step": 13542
+      "epoch": 6.0,
+      "eval_accuracy": 0.4512637482794966,
+      "eval_loss": 2.8827643394470215,
+      "eval_runtime": 72.7776,
+      "eval_samples_per_second": 834.061,
+      "eval_steps_per_second": 13.04,
+      "step": 13536
     },
     {
-      "epoch": 6.202580716619593,
-      "grad_norm": 0.30153268575668335,
-      "learning_rate": 0.00043746875000000003,
-      "loss": 2.754,
+      "epoch": 6.205673758865248,
+      "grad_norm": 0.29630789160728455,
+      "learning_rate": 0.0004375,
+      "loss": 2.759,
       "step": 14000
     },
     {
-      "epoch": 6.645622196378136,
-      "grad_norm": 0.29313963651657104,
+      "epoch": 6.648936170212766,
+      "grad_norm": 0.27569055557250977,
       "learning_rate": 0.00046871875,
-      "loss": 2.7377,
+      "loss": 2.7432,
       "step": 15000
     },
     {
-      "epoch": 6.999612338705211,
-      "eval_accuracy": 0.4545553537497972,
-      "eval_loss": 2.860243558883667,
-      "eval_runtime": 122.3572,
-      "eval_samples_per_second": 496.832,
-      "eval_steps_per_second": 7.764,
-      "step": 15799
+      "epoch": 7.0,
+      "eval_accuracy": 0.4541487994350967,
+      "eval_loss": 2.8589529991149902,
+      "eval_runtime": 72.8865,
+      "eval_samples_per_second": 832.815,
+      "eval_steps_per_second": 13.02,
+      "step": 15792
     },
     {
-      "epoch": 7.088663676136679,
-      "grad_norm": 0.2767232358455658,
+      "epoch": 7.092198581560283,
+      "grad_norm": 0.2677430808544159,
       "learning_rate": 0.00049996875,
-      "loss": 2.7295,
+      "loss": 2.7319,
       "step": 16000
     },
     {
-      "epoch": 7.531705155895221,
-      "grad_norm": 0.2507473826408386,
+      "epoch": 7.535460992907802,
+      "grad_norm": 0.2510625123977661,
       "learning_rate": 0.0005311875000000001,
-      "loss": 2.7014,
+      "loss": 2.7061,
       "step": 17000
     },
     {
-      "epoch": 7.974746635653763,
-      "grad_norm": 0.2542831301689148,
-      "learning_rate": 0.00056240625,
-      "loss": 2.7101,
+      "epoch": 7.9787234042553195,
+      "grad_norm": 0.23760418593883514,
+      "learning_rate": 0.0005624375,
+      "loss": 2.7146,
       "step": 18000
     },
     {
       "epoch": 8.0,
-      "eval_accuracy": 0.4571935151649448,
-      "eval_loss": 2.8389248847961426,
-      "eval_runtime": 121.9399,
-      "eval_samples_per_second": 498.532,
-      "eval_steps_per_second": 7.791,
-      "step": 18057
+      "eval_accuracy": 0.4565537086154539,
+      "eval_loss": 2.8377726078033447,
+      "eval_runtime": 72.7887,
+      "eval_samples_per_second": 833.934,
+      "eval_steps_per_second": 13.038,
+      "step": 18048
     },
     {
-      "epoch": 8.417788115412305,
-      "grad_norm": 0.2478516399860382,
+      "epoch": 8.421985815602836,
+      "grad_norm": 0.23823712766170502,
       "learning_rate": 0.00059365625,
-      "loss": 2.673,
+      "loss": 2.675,
       "step": 19000
     },
     {
-      "epoch": 8.860829595170848,
-      "grad_norm": 0.22448837757110596,
+      "epoch": 8.865248226950355,
+      "grad_norm": 0.23026619851589203,
       "learning_rate": 0.00062490625,
-      "loss": 2.684,
+      "loss": 2.6906,
       "step": 20000
     },
     {
-      "epoch": 8.99994461981503,
-      "eval_accuracy": 0.4586443233179834,
-      "eval_loss": 2.8260207176208496,
-      "eval_runtime": 122.3992,
-      "eval_samples_per_second": 496.662,
-      "eval_steps_per_second": 7.761,
-      "step": 20314
+      "epoch": 9.0,
+      "eval_accuracy": 0.45826760614791046,
+      "eval_loss": 2.8225581645965576,
+      "eval_runtime": 72.5363,
+      "eval_samples_per_second": 836.836,
+      "eval_steps_per_second": 13.083,
+      "step": 20304
     },
     {
-      "epoch": 9.30387107492939,
-      "grad_norm": 0.22841599583625793,
-      "learning_rate": 0.0006561562500000001,
+      "epoch": 9.308510638297872,
+      "grad_norm": 0.22488652169704437,
+      "learning_rate": 0.000656125,
       "loss": 2.6598,
       "step": 21000
     },
     {
-      "epoch": 9.746912554687933,
-      "grad_norm": 0.21708275377750397,
+      "epoch": 9.75177304964539,
+      "grad_norm": 0.20617271959781647,
       "learning_rate": 0.0006873749999999999,
-      "loss": 2.6654,
+      "loss": 2.6681,
       "step": 22000
     },
     {
-      "epoch": 9.999889239630061,
-      "eval_accuracy": 0.45958170407706767,
-      "eval_loss": 2.815507411956787,
-      "eval_runtime": 120.8572,
-      "eval_samples_per_second": 502.998,
-      "eval_steps_per_second": 7.861,
-      "step": 22571
+      "epoch": 10.0,
+      "eval_accuracy": 0.459518546549771,
+      "eval_loss": 2.813441753387451,
+      "eval_runtime": 72.4442,
+      "eval_samples_per_second": 837.9,
+      "eval_steps_per_second": 13.1,
+      "step": 22560
     },
     {
-      "epoch": 10.189954034446474,
-      "grad_norm": 0.2034858614206314,
+      "epoch": 10.195035460992909,
+      "grad_norm": 0.20597966015338898,
       "learning_rate": 0.000718625,
-      "loss": 2.6505,
+      "loss": 2.6559,
       "step": 23000
     },
     {
-      "epoch": 10.632995514205017,
-      "grad_norm": 0.20455513894557953,
-      "learning_rate": 0.000749875,
-      "loss": 2.6466,
+      "epoch": 10.638297872340425,
+      "grad_norm": 0.21323370933532715,
+      "learning_rate": 0.0007498437500000001,
+      "loss": 2.6498,
       "step": 24000
     },
     {
-      "epoch": 10.99983385944509,
-      "eval_accuracy": 0.4604201924885037,
-      "eval_loss": 2.807711601257324,
-      "eval_runtime": 121.2532,
-      "eval_samples_per_second": 501.356,
-      "eval_steps_per_second": 7.835,
-      "step": 24828
+      "epoch": 11.0,
+      "eval_accuracy": 0.4604723054276652,
+      "eval_loss": 2.8047826290130615,
+      "eval_runtime": 72.8612,
+      "eval_samples_per_second": 833.105,
+      "eval_steps_per_second": 13.025,
+      "step": 24816
     },
     {
-      "epoch": 11.07603699396356,
-      "grad_norm": 0.20884878933429718,
-      "learning_rate": 0.000781125,
-      "loss": 2.6497,
+      "epoch": 11.081560283687944,
+      "grad_norm": 0.21533997356891632,
+      "learning_rate": 0.00078109375,
+      "loss": 2.6487,
       "step": 25000
     },
     {
-      "epoch": 11.519078473722102,
-      "grad_norm": 0.2016923576593399,
-      "learning_rate": 0.00081234375,
-      "loss": 2.6304,
+      "epoch": 11.52482269503546,
+      "grad_norm": 0.21542951464653015,
+      "learning_rate": 0.0008123125,
+      "loss": 2.6323,
       "step": 26000
     },
     {
-      "epoch": 11.962119953480645,
-      "grad_norm": 0.1992848664522171,
-      "learning_rate": 0.00084359375,
-      "loss": 2.6474,
+      "epoch": 11.96808510638298,
+      "grad_norm": 0.19053979218006134,
+      "learning_rate": 0.0008435625,
+      "loss": 2.6497,
       "step": 27000
     },
     {
-      "epoch": 11.99977847926012,
-      "eval_accuracy": 0.4614518854538904,
-      "eval_loss": 2.802515983581543,
-      "eval_runtime": 121.3375,
-      "eval_samples_per_second": 501.007,
-      "eval_steps_per_second": 7.829,
-      "step": 27085
+      "epoch": 12.0,
+      "eval_accuracy": 0.46156735473880167,
+      "eval_loss": 2.7969932556152344,
+      "eval_runtime": 72.5419,
+      "eval_samples_per_second": 836.771,
+      "eval_steps_per_second": 13.082,
+      "step": 27072
     },
     {
-      "epoch": 12.405161433239186,
-      "grad_norm": 0.19233956933021545,
-      "learning_rate": 0.0008748125,
-      "loss": 2.6163,
+      "epoch": 12.411347517730496,
+      "grad_norm": 0.18872858583927155,
+      "learning_rate": 0.00087478125,
+      "loss": 2.6167,
       "step": 28000
     },
     {
-      "epoch": 12.84820291299773,
-      "grad_norm": 0.18767118453979492,
+      "epoch": 12.854609929078014,
+      "grad_norm": 0.1738893836736679,
       "learning_rate": 0.0009060312499999999,
-      "loss": 2.6366,
+      "loss": 2.6375,
       "step": 29000
     },
     {
-      "epoch": 12.999723099075151,
-      "eval_accuracy": 0.46189461094763445,
-      "eval_loss": 2.7982876300811768,
-      "eval_runtime": 121.2511,
-      "eval_samples_per_second": 501.364,
-      "eval_steps_per_second": 7.835,
-      "step": 29342
+      "epoch": 13.0,
+      "eval_accuracy": 0.4621078374843455,
+      "eval_loss": 2.7915232181549072,
+      "eval_runtime": 72.7216,
+      "eval_samples_per_second": 834.704,
+      "eval_steps_per_second": 13.05,
+      "step": 29328
     },
     {
-      "epoch": 13.291244392756273,
-      "grad_norm": 0.188876211643219,
-      "learning_rate": 0.00093728125,
-      "loss": 2.6143,
+      "epoch": 13.297872340425531,
+      "grad_norm": 0.1877707690000534,
+      "learning_rate": 0.00093725,
+      "loss": 2.6153,
       "step": 30000
     },
     {
-      "epoch": 13.734285872514814,
-      "grad_norm": 0.17773953080177307,
-      "learning_rate": 0.00096853125,
-      "loss": 2.625,
+      "epoch": 13.74113475177305,
+      "grad_norm": 0.186727836728096,
+      "learning_rate": 0.0009685000000000001,
+      "loss": 2.6278,
       "step": 31000
     },
     {
-      "epoch": 13.999667718890182,
-      "eval_accuracy": 0.46255660264467685,
-      "eval_loss": 2.792785406112671,
-      "eval_runtime": 120.7502,
-      "eval_samples_per_second": 503.444,
-      "eval_steps_per_second": 7.867,
-      "step": 31599
+      "epoch": 14.0,
+      "eval_accuracy": 0.462822946677559,
+      "eval_loss": 2.786189317703247,
+      "eval_runtime": 72.4588,
+      "eval_samples_per_second": 837.731,
+      "eval_steps_per_second": 13.097,
+      "step": 31584
     },
     {
-      "epoch": 14.177327352273357,
-      "grad_norm": 0.19869054853916168,
-      "learning_rate": 0.00099978125,
-      "loss": 2.6187,
+      "epoch": 14.184397163120567,
+      "grad_norm": 0.20270851254463196,
+      "learning_rate": 0.00099971875,
+      "loss": 2.6168,
       "step": 32000
     },
     {
-      "epoch": 14.620368832031899,
-      "grad_norm": 0.17133094370365143,
-      "learning_rate": 0.0009245053272450533,
-      "loss": 2.6109,
+      "epoch": 14.627659574468085,
+      "grad_norm": 0.17972639203071594,
+      "learning_rate": 0.0009244664634146341,
+      "loss": 2.6102,
       "step": 33000
     },
     {
-      "epoch": 14.99961233870521,
-      "eval_accuracy": 0.46537558287943165,
-      "eval_loss": 2.768995523452759,
-      "eval_runtime": 120.8755,
-      "eval_samples_per_second": 502.922,
-      "eval_steps_per_second": 7.859,
-      "step": 33856
+      "epoch": 15.0,
+      "eval_accuracy": 0.46529976086578023,
+      "eval_loss": 2.763315200805664,
+      "eval_runtime": 72.4263,
+      "eval_samples_per_second": 838.107,
+      "eval_steps_per_second": 13.103,
+      "step": 33840
     },
     {
-      "epoch": 15.063410311790442,
-      "grad_norm": 0.1772530972957611,
-      "learning_rate": 0.0008484018264840183,
-      "loss": 2.5987,
+      "epoch": 15.070921985815604,
+      "grad_norm": 0.17892582714557648,
+      "learning_rate": 0.0008483231707317073,
+      "loss": 2.6,
       "step": 34000
     },
     {
-      "epoch": 15.506451791548983,
-      "grad_norm": 0.17091083526611328,
-      "learning_rate": 0.0007723744292237444,
-      "loss": 2.5615,
+      "epoch": 15.51418439716312,
+      "grad_norm": 0.16611941158771515,
+      "learning_rate": 0.0007721036585365854,
+      "loss": 2.5602,
       "step": 35000
     },
     {
-      "epoch": 15.949493271307526,
-      "grad_norm": 0.17316913604736328,
-      "learning_rate": 0.0006962709284627093,
-      "loss": 2.5658,
+      "epoch": 15.957446808510639,
+      "grad_norm": 0.1676749587059021,
+      "learning_rate": 0.0006960365853658537,
+      "loss": 2.5668,
       "step": 36000
     },
     {
       "epoch": 16.0,
-      "eval_accuracy": 0.4686073564166006,
-      "eval_loss": 2.744506359100342,
-      "eval_runtime": 120.8269,
-      "eval_samples_per_second": 503.125,
-      "eval_steps_per_second": 7.862,
-      "step": 36114
-    },
-    {
-      "epoch": 16.39253475106607,
-      "grad_norm": 0.19304046034812927,
-      "learning_rate": 0.0006201674277016743,
-      "loss": 2.5123,
+      "eval_accuracy": 0.46856268478957125,
+      "eval_loss": 2.739426374435425,
+      "eval_runtime": 72.5666,
+      "eval_samples_per_second": 836.486,
+      "eval_steps_per_second": 13.078,
+      "step": 36096
+    },
+    {
+      "epoch": 16.400709219858157,
+      "grad_norm": 0.17558415234088898,
+      "learning_rate": 0.0006198170731707318,
+      "loss": 2.5115,
       "step": 37000
     },
     {
-      "epoch": 16.83557623082461,
-      "grad_norm": 0.17993681132793427,
-      "learning_rate": 0.0005440639269406394,
-      "loss": 2.5185,
+      "epoch": 16.843971631205672,
+      "grad_norm": 0.1874464899301529,
+      "learning_rate": 0.0005435975609756098,
+      "loss": 2.5178,
       "step": 38000
     },
     {
-      "epoch": 16.99994461981503,
-      "eval_accuracy": 0.47166972923301015,
-      "eval_loss": 2.72279953956604,
-      "eval_runtime": 121.0608,
-      "eval_samples_per_second": 502.152,
-      "eval_steps_per_second": 7.847,
-      "step": 38371
+      "epoch": 17.0,
+      "eval_accuracy": 0.47171248592021775,
+      "eval_loss": 2.7182633876800537,
+      "eval_runtime": 72.699,
+      "eval_samples_per_second": 834.964,
+      "eval_steps_per_second": 13.054,
+      "step": 38352
     },
     {
-      "epoch": 17.278617710583152,
-      "grad_norm": 0.18638047575950623,
-      "learning_rate": 0.0004680365296803653,
-      "loss": 2.4713,
+      "epoch": 17.28723404255319,
+      "grad_norm": 0.18552443385124207,
+      "learning_rate": 0.00046745426829268295,
+      "loss": 2.4731,
       "step": 39000
     },
     {
-      "epoch": 17.721659190341697,
-      "grad_norm": 0.19553589820861816,
-      "learning_rate": 0.0003919330289193303,
-      "loss": 2.4637,
+      "epoch": 17.73049645390071,
+      "grad_norm": 0.1822243332862854,
+      "learning_rate": 0.000391234756097561,
+      "loss": 2.462,
       "step": 40000
     },
     {
-      "epoch": 17.99988923963006,
-      "eval_accuracy": 0.4746745599919493,
-      "eval_loss": 2.7043166160583496,
-      "eval_runtime": 121.0615,
-      "eval_samples_per_second": 502.15,
-      "eval_steps_per_second": 7.847,
-      "step": 40628
+      "epoch": 18.0,
+      "eval_accuracy": 0.4747758459901975,
+      "eval_loss": 2.697719097137451,
+      "eval_runtime": 72.2911,
+      "eval_samples_per_second": 839.675,
+      "eval_steps_per_second": 13.127,
+      "step": 40608
     },
     {
-      "epoch": 18.16470067010024,
-      "grad_norm": 0.1946045607328415,
-      "learning_rate": 0.00031590563165905634,
-      "loss": 2.4315,
+      "epoch": 18.173758865248228,
+      "grad_norm": 0.18331420421600342,
+      "learning_rate": 0.000315015243902439,
+      "loss": 2.4319,
       "step": 41000
     },
     {
-      "epoch": 18.60774214985878,
-      "grad_norm": 0.1942271739244461,
-      "learning_rate": 0.0002398021308980213,
-      "loss": 2.3969,
+      "epoch": 18.617021276595743,
+      "grad_norm": 0.19214719533920288,
+      "learning_rate": 0.00023879573170731708,
+      "loss": 2.3974,
       "step": 42000
     },
     {
-      "epoch": 18.99983385944509,
-      "eval_accuracy": 0.4773752951691443,
-      "eval_loss": 2.689497947692871,
-      "eval_runtime": 121.1389,
-      "eval_samples_per_second": 501.829,
-      "eval_steps_per_second": 7.842,
-      "step": 42885
+      "epoch": 19.0,
+      "eval_accuracy": 0.4773433651479076,
+      "eval_loss": 2.683954954147339,
+      "eval_runtime": 72.4404,
+      "eval_samples_per_second": 837.944,
+      "eval_steps_per_second": 13.1,
+      "step": 42864
     },
     {
-      "epoch": 19.050783629617325,
-      "grad_norm": 0.1933259516954422,
-      "learning_rate": 0.0001636986301369863,
-      "loss": 2.3882,
+      "epoch": 19.06028368794326,
+      "grad_norm": 0.19565586745738983,
+      "learning_rate": 0.00016265243902439025,
+      "loss": 2.3863,
       "step": 43000
     },
     {
-      "epoch": 19.493825109375866,
-      "grad_norm": 0.1976754367351532,
-      "learning_rate": 8.75951293759513e-05,
-      "loss": 2.3278,
+      "epoch": 19.50354609929078,
+      "grad_norm": 0.19951286911964417,
+      "learning_rate": 8.643292682926828e-05,
+      "loss": 2.3273,
       "step": 44000
     },
     {
-      "epoch": 19.936866589134407,
-      "grad_norm": 0.1973201334476471,
-      "learning_rate": 1.1567732115677321e-05,
-      "loss": 2.3245,
+      "epoch": 19.9468085106383,
+      "grad_norm": 0.19867576658725739,
+      "learning_rate": 1.0213414634146342e-05,
+      "loss": 2.3259,
       "step": 45000
     },
     {
-      "epoch": 19.998892396300604,
-      "eval_accuracy": 0.47865689612852264,
-      "eval_loss": 2.6880221366882324,
-      "eval_runtime": 121.1537,
-      "eval_samples_per_second": 501.768,
-      "eval_steps_per_second": 7.841,
-      "step": 45140
+      "epoch": 20.0,
+      "eval_accuracy": 0.47877642614021604,
+      "eval_loss": 2.6820449829101562,
+      "eval_runtime": 72.6976,
+      "eval_samples_per_second": 834.98,
+      "eval_steps_per_second": 13.054,
+      "step": 45120
     },
     {
-      "epoch": 19.998892396300604,
-      "step": 45140,
-      "total_flos": 1.50966240067584e+18,
-      "train_loss": 2.8030111154835997,
-      "train_runtime": 43302.4979,
-      "train_samples_per_second": 266.866,
-      "train_steps_per_second": 1.042
+      "epoch": 20.0,
+      "step": 45120,
+      "total_flos": 1.50902942072832e+18,
+      "train_loss": 2.805498681169875,
+      "train_runtime": 30524.4807,
+      "train_samples_per_second": 378.402,
+      "train_steps_per_second": 1.478
     }
   ],
   "logging_steps": 1000,
-  "max_steps": 45140,
+  "max_steps": 45120,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -539,7 +539,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.50966240067584e+18,
+  "total_flos": 1.50902942072832e+18,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null