diff --git "a/checkpoint-42724/trainer_state.json" "b/checkpoint-42724/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-42724/trainer_state.json"
@@ -0,0 +1,6244 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "global_step": 42724,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1682242990654204e-06,
+      "loss": 10.8584,
+      "theoretical_loss": 20.812814784551147,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.841121495327103e-05,
+      "loss": 9.0184,
+      "theoretical_loss": 8.563479647615063,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011682242990654206,
+      "loss": 6.9386,
+      "theoretical_loss": 7.4777557010520255,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017523364485981307,
+      "loss": 6.096,
+      "theoretical_loss": 6.933751471898896,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023364485981308412,
+      "loss": 5.7275,
+      "theoretical_loss": 6.583563211430409,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029205607476635517,
+      "loss": 5.4399,
+      "theoretical_loss": 6.330710548120079,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035046728971962614,
+      "loss": 5.2491,
+      "theoretical_loss": 6.135526214944321,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004088785046728972,
+      "loss": 5.1116,
+      "theoretical_loss": 5.978098566873603,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046728971962616824,
+      "loss": 4.9855,
+      "theoretical_loss": 5.847114309269919,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499739928125591,
+      "loss": 4.8409,
+      "theoretical_loss": 5.73557379888612,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991488556837526,
+      "loss": 4.7327,
+      "theoretical_loss": 5.638867127075349,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985577832419141,
+      "loss": 4.6398,
+      "theoretical_loss": 5.553809364848902,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979667108000757,
+      "loss": 4.5391,
+      "theoretical_loss": 5.478115063560433,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973756383582371,
+      "loss": 4.4539,
+      "theoretical_loss": 5.410092942583357,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967845659163987,
+      "loss": 4.4011,
+      "theoretical_loss": 5.34845906673983,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961934934745603,
+      "loss": 4.2868,
+      "theoretical_loss": 5.292217549941562,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956024210327218,
+      "loss": 4.2131,
+      "theoretical_loss": 5.240581608773973,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950113485908833,
+      "loss": 4.1737,
+      "theoretical_loss": 5.192919707529784,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944202761490448,
+      "loss": 4.0132,
+      "theoretical_loss": 5.1487178463604355,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938292037072064,
+      "loss": 3.9839,
+      "theoretical_loss": 5.107552545409097,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932381312653678,
+      "loss": 3.8464,
+      "theoretical_loss": 5.069071100147241,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926470588235294,
+      "loss": 3.8002,
+      "theoretical_loss": 5.032976892842002,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492055986381691,
+      "loss": 3.716,
+      "theoretical_loss": 4.999018291228659,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914649139398525,
+      "loss": 3.7048,
+      "theoretical_loss": 4.966980138355957,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490873841498014,
+      "loss": 3.6445,
+      "theoretical_loss": 4.936677144629131,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902827690561755,
+      "loss": 3.6049,
+      "theoretical_loss": 4.907948696834077,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896916966143371,
+      "loss": 3.5803,
+      "theoretical_loss": 4.880654736816922,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891006241724985,
+      "loss": 3.5116,
+      "theoretical_loss": 4.8546724574857745,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885095517306601,
+      "loss": 3.4969,
+      "theoretical_loss": 4.829893630312808,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879184792888217,
+      "loss": 3.4364,
+      "theoretical_loss": 4.806222425781916,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873274068469832,
+      "loss": 3.444,
+      "theoretical_loss": 4.783573622280253,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048673633440514467,
+      "loss": 3.4411,
+      "theoretical_loss": 4.7618711237764035,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861452619633062,
+      "loss": 3.448,
+      "theoretical_loss": 4.741046724966468,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855541895214677,
+      "loss": 3.3972,
+      "theoretical_loss": 4.721039076253046,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849631170796293,
+      "loss": 3.4331,
+      "theoretical_loss": 4.701792811235861,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843720446377908,
+      "loss": 3.4396,
+      "theoretical_loss": 4.683257807239925,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048378097219595233,
+      "loss": 3.4205,
+      "theoretical_loss": 4.665388555430277,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048318989975411385,
+      "loss": 3.3658,
+      "theoretical_loss": 4.648143621723734,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048259882731227537,
+      "loss": 3.3609,
+      "theoretical_loss": 4.631485183343639,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820077548704369,
+      "loss": 3.3609,
+      "theoretical_loss": 4.6153786287197125,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141668242859847,
+      "loss": 3.3415,
+      "theoretical_loss": 4.5997922106945,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048082560998676,
+      "loss": 3.3173,
+      "theoretical_loss": 4.58469674479667,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802345375449215,
+      "loss": 3.3429,
+      "theoretical_loss": 4.570065345782512,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047964346510308303,
+      "loss": 3.2678,
+      "theoretical_loss": 4.555873196808033,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905239266124455,
+      "loss": 3.2872,
+      "theoretical_loss": 4.542097346534794,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047846132021940607,
+      "loss": 3.2486,
+      "theoretical_loss": 4.528716530238812,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778702477775676,
+      "loss": 3.2852,
+      "theoretical_loss": 4.515711011618992,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047727917533572917,
+      "loss": 3.2308,
+      "theoretical_loss": 4.503062442517334,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766881028938907,
+      "loss": 3.2673,
+      "theoretical_loss": 4.4907537381892615,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760970304520522,
+      "loss": 3.2672,
+      "theoretical_loss": 4.478768966115963,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 101104,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.6775944232940674,
+      "objective/train/theoretical_loss": 4.467093246645215,
+      "objective/train/tokens_used": 184300000,
+      "theoretical_loss": 4.467093246645215,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047550595801021373,
+      "loss": 3.2026,
+      "theoretical_loss": 4.467093246645215,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047491488556837525,
+      "loss": 3.2085,
+      "theoretical_loss": 4.455712663993541,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432381312653677,
+      "loss": 3.2025,
+      "theoretical_loss": 4.444614186349425,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047373274068469835,
+      "loss": 3.183,
+      "theoretical_loss": 4.433785593991642,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047314166824285987,
+      "loss": 3.1657,
+      "theoretical_loss": 4.423215414484155,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725505958010214,
+      "loss": 3.1526,
+      "theoretical_loss": 4.412892864134137,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719595233591829,
+      "loss": 3.1314,
+      "theoretical_loss": 4.402807795006074,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047136845091734443,
+      "loss": 3.1306,
+      "theoretical_loss": 4.392950646875857,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047077737847550595,
+      "loss": 3.1734,
+      "theoretical_loss": 4.383312403586528,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701863060336675,
+      "loss": 3.1533,
+      "theoretical_loss": 4.373884553334271,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046959523359182905,
+      "loss": 3.1653,
+      "theoretical_loss": 4.364659052470699,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046900416114999057,
+      "loss": 3.1275,
+      "theoretical_loss": 4.355628292457278,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684130887081521,
+      "loss": 3.1234,
+      "theoretical_loss": 4.346785069650667,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678220162663136,
+      "loss": 3.1054,
+      "theoretical_loss": 4.338122557635156,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046723094382447513,
+      "loss": 3.1138,
+      "theoretical_loss": 4.329634281850807,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046663987138263665,
+      "loss": 3.1681,
+      "theoretical_loss": 4.321314096294248,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660487989407982,
+      "loss": 3.0757,
+      "theoretical_loss": 4.3131561620937875,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046545772649895975,
+      "loss": 3.0927,
+      "theoretical_loss": 4.305154927782223,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046486665405712127,
+      "loss": 3.0431,
+      "theoretical_loss": 4.297305111109683,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642755816152828,
+      "loss": 3.106,
+      "theoretical_loss": 4.2896016822555945,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636845091734443,
+      "loss": 3.1308,
+      "theoretical_loss": 4.282039848313611,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046309343673160583,
+      "loss": 3.122,
+      "theoretical_loss": 4.274615038936293,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625023642897674,
+      "loss": 3.1477,
+      "theoretical_loss": 4.267322893037893,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619112918479289,
+      "loss": 3.1332,
+      "theoretical_loss": 4.26015924646374,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046132021940609044,
+      "loss": 3.1128,
+      "theoretical_loss": 4.253120120543809,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046072914696425197,
+      "loss": 3.0438,
+      "theoretical_loss": 4.24620171145605,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601380745224135,
+      "loss": 3.0475,
+      "theoretical_loss": 4.239400380332256,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459547002080575,
+      "loss": 3.0507,
+      "theoretical_loss": 4.232712644045627,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589559296387366,
+      "loss": 3.0612,
+      "theoretical_loss": 4.226135166624862,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583648571968981,
+      "loss": 3.0783,
+      "theoretical_loss": 4.21966475124477,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577737847550596,
+      "loss": 3.0221,
+      "theoretical_loss": 4.21329833274792,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571827123132211,
+      "loss": 3.0286,
+      "theoretical_loss": 4.207032970655965,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565916398713826,
+      "loss": 2.974,
+      "theoretical_loss": 4.2008658426329974,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045600056742954413,
+      "loss": 2.995,
+      "theoretical_loss": 4.194794238366544,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554094949877057,
+      "loss": 2.9599,
+      "theoretical_loss": 4.188815553834879,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045481842254586723,
+      "loss": 2.9159,
+      "theoretical_loss": 4.182927285931959,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045422735010402875,
+      "loss": 2.9718,
+      "theoretical_loss": 4.177127027423776,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045363627766219027,
+      "loss": 3.0047,
+      "theoretical_loss": 4.171412462212087,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004530452052203518,
+      "loss": 2.9699,
+      "theoretical_loss": 4.165781360883513,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004524541327785133,
+      "loss": 2.9841,
+      "theoretical_loss": 4.160231576523763,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045186306033667483,
+      "loss": 3.0051,
+      "theoretical_loss": 4.1547610407784346,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004512719878948364,
+      "loss": 3.0277,
+      "theoretical_loss": 4.1493677601432815,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045068091545299793,
+      "loss": 3.0073,
+      "theoretical_loss": 4.1440498124682446,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045008984301115945,
+      "loss": 3.0092,
+      "theoretical_loss": 4.138805343660737,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044949877056932097,
+      "loss": 2.9717,
+      "theoretical_loss": 4.133632564574831,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004489076981274825,
+      "loss": 2.9231,
+      "theoretical_loss": 4.128529748073999,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000448316625685644,
+      "loss": 2.9461,
+      "theoretical_loss": 4.123495226256027,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004477255532438056,
+      "loss": 2.9478,
+      "theoretical_loss": 4.118527387829552,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471344808019671,
+      "loss": 2.9396,
+      "theoretical_loss": 4.11362467563246,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044654340836012863,
+      "loss": 2.9153,
+      "theoretical_loss": 4.108785584283144,
+      "tokens_seen": 324403200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5369745433205646,
+      "debugging/distinct-1-grams": 0.7852136036914666,
+      "debugging/distinct-2-grams": 0.975898791228072,
+      "debugging/entropy-1-grams": 5.9202049103095575,
+      "debugging/entropy-2-grams": 6.8314825534787165,
+      "debugging/length": 575.3333333333334,
+      "debugging/num_segments": 12,
+      "epoch": 0.12,
+      "objective/train/docs_used": 191544,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.262237310409546,
+      "objective/train/theoretical_loss": 4.104008657956216,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.104008657956216,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044595233591829015,
+      "loss": 2.92,
+      "theoretical_loss": 4.104008657956216,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044536126347645167,
+      "loss": 2.9517,
+      "theoretical_loss": 4.099292488274917,
+      "tokens_seen": 330956800
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004447701910346132,
+      "loss": 2.9843,
+      "theoretical_loss": 4.094635712313026,
+      "tokens_seen": 334233600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044417911859277476,
+      "loss": 2.9337,
+      "theoretical_loss": 4.090037010699552,
+      "tokens_seen": 337510400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435880461509363,
+      "loss": 2.9564,
+      "theoretical_loss": 4.085495105819987,
+      "tokens_seen": 340787200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004429969737090978,
+      "loss": 2.9046,
+      "theoretical_loss": 4.081008760108329,
+      "tokens_seen": 344064000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004424059012672593,
+      "loss": 2.922,
+      "theoretical_loss": 4.076576774424465,
+      "tokens_seen": 347340800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044181482882542085,
+      "loss": 2.9275,
+      "theoretical_loss": 4.072197986511911,
+      "tokens_seen": 350617600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044122375638358237,
+      "loss": 2.9031,
+      "theoretical_loss": 4.067871269531193,
+      "tokens_seen": 353894400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004406326839417439,
+      "loss": 2.9345,
+      "theoretical_loss": 4.063595530664515,
+      "tokens_seen": 357171200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044004161149990546,
+      "loss": 2.9301,
+      "theoretical_loss": 4.059369709787625,
+      "tokens_seen": 360448000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000439450539058067,
+      "loss": 2.9386,
+      "theoretical_loss": 4.055192778205064,
+      "tokens_seen": 363724800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388594666162285,
+      "loss": 2.9477,
+      "theoretical_loss": 4.051063737445231,
+      "tokens_seen": 367001600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043826839417439,
+      "loss": 2.8887,
+      "theoretical_loss": 4.046981618111936,
+      "tokens_seen": 370278400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043767732173255155,
+      "loss": 2.9278,
+      "theoretical_loss": 4.042945478789308,
+      "tokens_seen": 373555200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043708624929071307,
+      "loss": 2.9088,
+      "theoretical_loss": 4.038954404997148,
+      "tokens_seen": 376832000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043649517684887464,
+      "loss": 2.882,
+      "theoretical_loss": 4.035007508193978,
+      "tokens_seen": 380108800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043590410440703616,
+      "loss": 2.9125,
+      "theoretical_loss": 4.031103924825214,
+      "tokens_seen": 383385600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004353130319651977,
+      "loss": 2.8289,
+      "theoretical_loss": 4.027242815414074,
+      "tokens_seen": 386662400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004347219595233592,
+      "loss": 2.9065,
+      "theoretical_loss": 4.023423363692939,
+      "tokens_seen": 389939200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004341308870815207,
+      "loss": 2.9067,
+      "theoretical_loss": 4.019644775773043,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043353981463968225,
+      "loss": 2.8752,
+      "theoretical_loss": 4.015906279350517,
+      "tokens_seen": 396492800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004329487421978438,
+      "loss": 2.8745,
+      "theoretical_loss": 4.01220712294689,
+      "tokens_seen": 399769600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043235766975600534,
+      "loss": 2.903,
+      "theoretical_loss": 4.008546575182286,
+      "tokens_seen": 403046400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043176659731416686,
+      "loss": 2.9016,
+      "theoretical_loss": 4.0049239240796695,
+      "tokens_seen": 406323200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004311755248723284,
+      "loss": 2.911,
+      "theoretical_loss": 4.001338476398553,
+      "tokens_seen": 409600000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004305844524304899,
+      "loss": 2.8983,
+      "theoretical_loss": 3.9977895569967217,
+      "tokens_seen": 412876800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004299933799886514,
+      "loss": 2.9451,
+      "theoretical_loss": 3.99427650821855,
+      "tokens_seen": 416153600
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042940230754681295,
+      "loss": 2.9276,
+      "theoretical_loss": 3.990798689308642,
+      "tokens_seen": 419430400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004288112351049745,
+      "loss": 2.9441,
+      "theoretical_loss": 3.987355475849527,
+      "tokens_seen": 422707200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000428220162663136,
+      "loss": 2.8943,
+      "theoretical_loss": 3.98394625922226,
+      "tokens_seen": 425984000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004276290902212975,
+      "loss": 2.9401,
+      "theoretical_loss": 3.9805704460888256,
+      "tokens_seen": 429260800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042703801777945903,
+      "loss": 2.9437,
+      "theoretical_loss": 3.9772274578953013,
+      "tokens_seen": 432537600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042644694533762055,
+      "loss": 2.9373,
+      "theoretical_loss": 3.973916730394796,
+      "tokens_seen": 435814400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042585587289578207,
+      "loss": 2.9168,
+      "theoretical_loss": 3.970637713189244,
+      "tokens_seen": 439091200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042526480045394365,
+      "loss": 2.8928,
+      "theoretical_loss": 3.967389869289161,
+      "tokens_seen": 442368000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042467372801210517,
+      "loss": 2.9101,
+      "theoretical_loss": 3.964172674690542,
+      "tokens_seen": 445644800
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004240826555702667,
+      "loss": 2.8902,
+      "theoretical_loss": 3.9609856179681078,
+      "tokens_seen": 448921600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004234915831284282,
+      "loss": 2.8893,
+      "theoretical_loss": 3.957828199884155,
+      "tokens_seen": 452198400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042290051068658973,
+      "loss": 2.8546,
+      "theoretical_loss": 3.9546999330123036,
+      "tokens_seen": 455475200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042230943824475125,
+      "loss": 2.917,
+      "theoretical_loss": 3.951600341375469,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004217183658029128,
+      "loss": 2.866,
+      "theoretical_loss": 3.9485289600974305,
+      "tokens_seen": 462028800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042112729336107435,
+      "loss": 2.8189,
+      "theoretical_loss": 3.945485335067386,
+      "tokens_seen": 465305600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042053622091923587,
+      "loss": 2.8606,
+      "theoretical_loss": 3.9424690226169212,
+      "tokens_seen": 468582400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199451484773974,
+      "loss": 2.8887,
+      "theoretical_loss": 3.9394795892088585,
+      "tokens_seen": 471859200
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004193540760355589,
+      "loss": 2.8486,
+      "theoretical_loss": 3.9365166111374608,
+      "tokens_seen": 475136000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041876300359372043,
+      "loss": 2.7856,
+      "theoretical_loss": 3.933579674239507,
+      "tokens_seen": 478412800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041817193115188195,
+      "loss": 2.8535,
+      "theoretical_loss": 3.930668373615765,
+      "tokens_seen": 481689600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004175808587100435,
+      "loss": 2.7963,
+      "theoretical_loss": 3.9277823133624366,
+      "tokens_seen": 484966400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041698978626820505,
+      "loss": 2.8532,
+      "theoretical_loss": 3.9249211063121283,
+      "tokens_seen": 488243200
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 280311,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.946796417236328,
+      "objective/train/theoretical_loss": 3.9220843737839752,
+      "objective/train/tokens_used": 511980000,
+      "theoretical_loss": 3.9220843737839752,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041639871382636657,
+      "loss": 2.8478,
+      "theoretical_loss": 3.9220843737839752,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004158076413845281,
+      "loss": 2.8832,
+      "theoretical_loss": 3.919271745342514,
+      "tokens_seen": 494796800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004152165689426896,
+      "loss": 2.8583,
+      "theoretical_loss": 3.916482858564957,
+      "tokens_seen": 498073600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041462549650085113,
+      "loss": 2.7761,
+      "theoretical_loss": 3.9137173588165135,
+      "tokens_seen": 501350400
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004140344240590127,
+      "loss": 2.8483,
+      "theoretical_loss": 3.9109748990334348,
+      "tokens_seen": 504627200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004134433516171742,
+      "loss": 2.8321,
+      "theoretical_loss": 3.908255139513467,
+      "tokens_seen": 507904000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041285227917533575,
+      "loss": 2.8297,
+      "theoretical_loss": 3.905557747713412,
+      "tokens_seen": 511180800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041226120673349727,
+      "loss": 2.7956,
+      "theoretical_loss": 3.90288239805351,
+      "tokens_seen": 514457600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004116701342916588,
+      "loss": 2.7937,
+      "theoretical_loss": 3.9002287717283783,
+      "tokens_seen": 517734400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004110790618498203,
+      "loss": 2.7995,
+      "theoretical_loss": 3.8975965565242423,
+      "tokens_seen": 521011200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004104879894079819,
+      "loss": 2.7679,
+      "theoretical_loss": 3.894985446642206,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004098969169661434,
+      "loss": 2.7851,
+      "theoretical_loss": 3.8923951425273398,
+      "tokens_seen": 527564800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004093058445243049,
+      "loss": 2.7783,
+      "theoretical_loss": 3.889825350703344,
+      "tokens_seen": 530841600
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040871477208246645,
+      "loss": 2.8113,
+      "theoretical_loss": 3.8872757836125844,
+      "tokens_seen": 534118400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040812369964062797,
+      "loss": 2.7985,
+      "theoretical_loss": 3.8847461594612884,
+      "tokens_seen": 537395200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004075326271987895,
+      "loss": 2.8391,
+      "theoretical_loss": 3.882236202069703,
+      "tokens_seen": 540672000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000406941554756951,
+      "loss": 2.8309,
+      "theoretical_loss": 3.8797456407270343,
+      "tokens_seen": 543948800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004063504823151126,
+      "loss": 2.8188,
+      "theoretical_loss": 3.8772742100509774,
+      "tokens_seen": 547225600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004057594098732741,
+      "loss": 2.8115,
+      "theoretical_loss": 3.874821649851678,
+      "tokens_seen": 550502400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004051683374314356,
+      "loss": 2.8505,
+      "theoretical_loss": 3.8723877049999444,
+      "tokens_seen": 553779200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040457726498959715,
+      "loss": 2.7899,
+      "theoretical_loss": 3.869972125299568,
+      "tokens_seen": 557056000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040398619254775867,
+      "loss": 2.8405,
+      "theoretical_loss": 3.867574665363595,
+      "tokens_seen": 560332800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004033951201059202,
+      "loss": 2.8184,
+      "theoretical_loss": 3.865195084494398,
+      "tokens_seen": 563609600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040280404766408176,
+      "loss": 2.8252,
+      "theoretical_loss": 3.8628331465674224,
+      "tokens_seen": 566886400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004022129752222433,
+      "loss": 2.8865,
+      "theoretical_loss": 3.860488619918462,
+      "tokens_seen": 570163200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004016219027804048,
+      "loss": 2.8926,
+      "theoretical_loss": 3.858161277234349,
+      "tokens_seen": 573440000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004010308303385663,
+      "loss": 2.8499,
+      "theoretical_loss": 3.855850895446925,
+      "tokens_seen": 576716800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040043975789672785,
+      "loss": 2.8563,
+      "theoretical_loss": 3.8535572556301823,
+      "tokens_seen": 579993600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003998486854548893,
+      "loss": 2.8469,
+      "theoretical_loss": 3.851280142900463,
+      "tokens_seen": 583270400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003992576130130509,
+      "loss": 2.8482,
+      "theoretical_loss": 3.8490193463196127,
+      "tokens_seen": 586547200
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003986665405712124,
+      "loss": 2.8413,
+      "theoretical_loss": 3.8467746588009692,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039807546812937393,
+      "loss": 2.846,
+      "theoretical_loss": 3.8445458770181116,
+      "tokens_seen": 593100800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039748439568753545,
+      "loss": 2.8096,
+      "theoretical_loss": 3.842332801316254,
+      "tokens_seen": 596377600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039689332324569697,
+      "loss": 2.81,
+      "theoretical_loss": 3.840135235626204,
+      "tokens_seen": 599654400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003963022508038585,
+      "loss": 2.7778,
+      "theoretical_loss": 3.8379529873807945,
+      "tokens_seen": 602931200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039571117836202,
+      "loss": 2.8046,
+      "theoretical_loss": 3.835785867433705,
+      "tokens_seen": 606208000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003951201059201816,
+      "loss": 2.7609,
+      "theoretical_loss": 3.8336336899805916,
+      "tokens_seen": 609484800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003945290334783431,
+      "loss": 2.7938,
+      "theoretical_loss": 3.8314962724824495,
+      "tokens_seen": 612761600
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039393796103650463,
+      "loss": 2.8098,
+      "theoretical_loss": 3.8293734355911297,
+      "tokens_seen": 616038400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039334688859466615,
+      "loss": 2.7622,
+      "theoretical_loss": 3.8272650030769433,
+      "tokens_seen": 619315200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039275581615282767,
+      "loss": 2.7791,
+      "theoretical_loss": 3.8251708017582793,
+      "tokens_seen": 622592000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003921647437109892,
+      "loss": 2.7272,
+      "theoretical_loss": 3.8230906614331737,
+      "tokens_seen": 625868800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039157367126915077,
+      "loss": 2.7712,
+      "theoretical_loss": 3.821024414812761,
+      "tokens_seen": 629145600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003909825988273123,
+      "loss": 2.7899,
+      "theoretical_loss": 3.818971897456552,
+      "tokens_seen": 632422400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003903915263854738,
+      "loss": 2.8311,
+      "theoretical_loss": 3.8169329477094784,
+      "tokens_seen": 635699200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038980045394363533,
+      "loss": 2.8424,
+      "theoretical_loss": 3.814907406640639,
+      "tokens_seen": 638976000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038920938150179685,
+      "loss": 2.8391,
+      "theoretical_loss": 3.8128951179837056,
+      "tokens_seen": 642252800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038861830905995837,
+      "loss": 2.8258,
+      "theoretical_loss": 3.8108959280789243,
+      "tokens_seen": 645529600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038802723661811994,
+      "loss": 2.8064,
+      "theoretical_loss": 3.8089096858166718,
+      "tokens_seen": 648806400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038743616417628147,
+      "loss": 2.8018,
+      "theoretical_loss": 3.8069362425825037,
+      "tokens_seen": 652083200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5495775421806781,
+      "debugging/distinct-1-grams": 0.7858205702842564,
+      "debugging/distinct-2-grams": 0.9509135221332653,
+      "debugging/entropy-1-grams": 6.135781796217266,
+      "debugging/entropy-2-grams": 7.025177223786267,
+      "debugging/length": 509.3888888888889,
+      "debugging/num_segments": 18,
+      "epoch": 0.23,
+      "objective/train/docs_used": 369170,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.964120864868164,
+      "objective/train/theoretical_loss": 3.8049754522036645,
+      "objective/train/tokens_used": 675820000,
+      "theoretical_loss": 3.8049754522036645,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000386845091734443,
+      "loss": 2.7933,
+      "theoretical_loss": 3.8049754522036645,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003862540192926045,
+      "loss": 2.8017,
+      "theoretical_loss": 3.8030271708970003,
+      "tokens_seen": 658636800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038566294685076603,
+      "loss": 2.7857,
+      "theoretical_loss": 3.801091257218237,
+      "tokens_seen": 661913600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038507187440892755,
+      "loss": 2.7687,
+      "theoretical_loss": 3.79916757201258,
+      "tokens_seen": 665190400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003844926234159259,
+      "loss": 2.7757,
+      "theoretical_loss": 3.7972559783665965,
+      "tokens_seen": 668467200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003839015509740874,
+      "loss": 2.829,
+      "theoretical_loss": 3.7953563415613325,
+      "tokens_seen": 671744000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833104785322489,
+      "loss": 2.8231,
+      "theoretical_loss": 3.7934685290266454,
+      "tokens_seen": 675020800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038271940609041044,
+      "loss": 2.8413,
+      "theoretical_loss": 3.7915924102966914,
+      "tokens_seen": 678297600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038212833364857196,
+      "loss": 2.8301,
+      "theoretical_loss": 3.789727856966552,
+      "tokens_seen": 681574400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003815372612067335,
+      "loss": 2.8133,
+      "theoretical_loss": 3.787874742649958,
+      "tokens_seen": 684851200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000380946188764895,
+      "loss": 2.8341,
+      "theoretical_loss": 3.786032942938073,
+      "tokens_seen": 688128000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003803551163230566,
+      "loss": 2.8079,
+      "theoretical_loss": 3.784202335359316,
+      "tokens_seen": 691404800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003797640438812181,
+      "loss": 2.812,
+      "theoretical_loss": 3.7823827993401844,
+      "tokens_seen": 694681600
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003791729714393796,
+      "loss": 2.7577,
+      "theoretical_loss": 3.7805742161670466,
+      "tokens_seen": 697958400
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037858189899754114,
+      "loss": 2.7862,
+      "theoretical_loss": 3.7787764689488847,
+      "tokens_seen": 701235200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037799082655570266,
+      "loss": 2.7499,
+      "theoretical_loss": 3.77698944258095,
+      "tokens_seen": 704512000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003773997541138642,
+      "loss": 2.7967,
+      "theoretical_loss": 3.7752130237093064,
+      "tokens_seen": 707788800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037680868167202575,
+      "loss": 2.7918,
+      "theoretical_loss": 3.773447100696245,
+      "tokens_seen": 711065600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003762176092301873,
+      "loss": 2.8046,
+      "theoretical_loss": 3.771691563586529,
+      "tokens_seen": 714342400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003756265367883488,
+      "loss": 2.7732,
+      "theoretical_loss": 3.7699463040744616,
+      "tokens_seen": 717619200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003750354643465103,
+      "loss": 2.7373,
+      "theoretical_loss": 3.768211215471741,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037444439190467184,
+      "loss": 2.7512,
+      "theoretical_loss": 3.766486192676084,
+      "tokens_seen": 724172800
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037385331946283336,
+      "loss": 2.7115,
+      "theoretical_loss": 3.764771132140602,
+      "tokens_seen": 727449600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037326224702099493,
+      "loss": 2.7214,
+      "theoretical_loss": 3.763065931843898,
+      "tokens_seen": 730726400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037267117457915645,
+      "loss": 2.7296,
+      "theoretical_loss": 3.7613704912608723,
+      "tokens_seen": 734003200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000372080102137318,
+      "loss": 2.7715,
+      "theoretical_loss": 3.759684711334213,
+      "tokens_seen": 737280000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003714890296954795,
+      "loss": 2.794,
+      "theoretical_loss": 3.7580084944465555,
+      "tokens_seen": 740556800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000370897957253641,
+      "loss": 2.785,
+      "theoretical_loss": 3.7563417443932905,
+      "tokens_seen": 743833600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037030688481180254,
+      "loss": 2.7882,
+      "theoretical_loss": 3.754684366355999,
+      "tokens_seen": 747110400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036971581236996406,
+      "loss": 2.7543,
+      "theoretical_loss": 3.753036266876505,
+      "tokens_seen": 750387200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036912473992812563,
+      "loss": 2.7509,
+      "theoretical_loss": 3.751397353831524,
+      "tokens_seen": 753664000
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036853366748628715,
+      "loss": 2.7483,
+      "theoretical_loss": 3.749767536407891,
+      "tokens_seen": 756940800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679425950444487,
+      "loss": 2.7265,
+      "theoretical_loss": 3.7481467250783504,
+      "tokens_seen": 760217600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003673515226026102,
+      "loss": 2.7443,
+      "theoretical_loss": 3.746534831577904,
+      "tokens_seen": 763494400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003667604501607717,
+      "loss": 2.6939,
+      "theoretical_loss": 3.744931768880681,
+      "tokens_seen": 766771200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036616937771893324,
+      "loss": 2.7158,
+      "theoretical_loss": 3.743337451177343,
+      "tokens_seen": 770048000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003655783052770948,
+      "loss": 2.7014,
+      "theoretical_loss": 3.74175179385298,
+      "tokens_seen": 773324800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036498723283525633,
+      "loss": 2.7803,
+      "theoretical_loss": 3.740174713465512,
+      "tokens_seen": 776601600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036439616039341785,
+      "loss": 2.7231,
+      "theoretical_loss": 3.7386061277245655,
+      "tokens_seen": 779878400
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003638050879515794,
+      "loss": 2.7733,
+      "theoretical_loss": 3.7370459554708146,
+      "tokens_seen": 783155200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003632140155097409,
+      "loss": 2.804,
+      "theoretical_loss": 3.735494116655784,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003626229430679024,
+      "loss": 2.7704,
+      "theoretical_loss": 3.733950532322087,
+      "tokens_seen": 789708800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000362031870626064,
+      "loss": 2.7873,
+      "theoretical_loss": 3.7324151245841044,
+      "tokens_seen": 792985600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036144079818422546,
+      "loss": 2.7787,
+      "theoretical_loss": 3.730887816609077,
+      "tokens_seen": 796262400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000360849725742387,
+      "loss": 2.7795,
+      "theoretical_loss": 3.729368532598609,
+      "tokens_seen": 799539200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003602586533005485,
+      "loss": 2.7645,
+      "theoretical_loss": 3.7278571977705734,
+      "tokens_seen": 802816000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035966758085871,
+      "loss": 2.7904,
+      "theoretical_loss": 3.7263537383414023,
+      "tokens_seen": 806092800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035907650841687154,
+      "loss": 2.7791,
+      "theoretical_loss": 3.7248580815087626,
+      "tokens_seen": 809369600
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035848543597503306,
+      "loss": 2.7984,
+      "theoretical_loss": 3.7233701554345924,
+      "tokens_seen": 812646400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035790618498203144,
+      "loss": 2.7796,
+      "theoretical_loss": 3.721889889228506,
+      "tokens_seen": 815923200
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 458185,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.0822372436523438,
+      "objective/train/theoretical_loss": 3.720417212931543,
+      "objective/train/tokens_used": 839660000,
+      "theoretical_loss": 3.720417212931543,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003573269339890297,
+      "loss": 2.761,
+      "theoretical_loss": 3.720417212931543,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035673586154719123,
+      "loss": 2.7429,
+      "theoretical_loss": 3.7189520575002666,
+      "tokens_seen": 822476800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035614478910535275,
+      "loss": 2.7242,
+      "theoretical_loss": 3.717494354791188,
+      "tokens_seen": 825753600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003555537166635143,
+      "loss": 2.7352,
+      "theoretical_loss": 3.716044037545523,
+      "tokens_seen": 829030400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003549626442216758,
+      "loss": 2.7653,
+      "theoretical_loss": 3.714601039374263,
+      "tokens_seen": 832307200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035437157177983737,
+      "loss": 2.7024,
+      "theoretical_loss": 3.7131652947435536,
+      "tokens_seen": 835584000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003537804993379989,
+      "loss": 2.7041,
+      "theoretical_loss": 3.7117367389603793,
+      "tokens_seen": 838860800
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003531894268961604,
+      "loss": 2.7092,
+      "theoretical_loss": 3.710315308158541,
+      "tokens_seen": 842137600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035259835445432193,
+      "loss": 2.7506,
+      "theoretical_loss": 3.7089009392849173,
+      "tokens_seen": 845414400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035200728201248345,
+      "loss": 2.7066,
+      "theoretical_loss": 3.7074935700860143,
+      "tokens_seen": 848691200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000351416209570645,
+      "loss": 2.7087,
+      "theoretical_loss": 3.706093139094781,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035082513712880655,
+      "loss": 2.7168,
+      "theoretical_loss": 3.7046995856176954,
+      "tokens_seen": 855244800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035023406468696807,
+      "loss": 2.76,
+      "theoretical_loss": 3.703312849722111,
+      "tokens_seen": 858521600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003496429922451296,
+      "loss": 2.7042,
+      "theoretical_loss": 3.701932872223858,
+      "tokens_seen": 861798400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003490519198032911,
+      "loss": 2.6904,
+      "theoretical_loss": 3.7005595946750924,
+      "tokens_seen": 865075200
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034846084736145263,
+      "loss": 2.7039,
+      "theoretical_loss": 3.699192959352386,
+      "tokens_seen": 868352000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034786977491961415,
+      "loss": 2.6842,
+      "theoretical_loss": 3.6978329092450557,
+      "tokens_seen": 871628800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034727870247777573,
+      "loss": 2.7035,
+      "theoretical_loss": 3.6964793880437226,
+      "tokens_seen": 874905600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034668763003593725,
+      "loss": 2.7266,
+      "theoretical_loss": 3.6951323401290974,
+      "tokens_seen": 878182400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034609655759409877,
+      "loss": 2.7197,
+      "theoretical_loss": 3.6937917105609834,
+      "tokens_seen": 881459200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003455054851522603,
+      "loss": 2.7329,
+      "theoretical_loss": 3.692457445067501,
+      "tokens_seen": 884736000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003449144127104218,
+      "loss": 2.735,
+      "theoretical_loss": 3.6911294900345166,
+      "tokens_seen": 888012800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034432334026858333,
+      "loss": 2.6776,
+      "theoretical_loss": 3.6898077924952775,
+      "tokens_seen": 891289600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034373226782674485,
+      "loss": 2.725,
+      "theoretical_loss": 3.6884923001202505,
+      "tokens_seen": 894566400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034314119538490643,
+      "loss": 2.6866,
+      "theoretical_loss": 3.6871829612071583,
+      "tokens_seen": 897843200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003425501229430679,
+      "loss": 2.6789,
+      "theoretical_loss": 3.6858797246711976,
+      "tokens_seen": 901120000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003419590505012294,
+      "loss": 2.6709,
+      "theoretical_loss": 3.684582540035456,
+      "tokens_seen": 904396800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034136797805939094,
+      "loss": 2.6846,
+      "theoretical_loss": 3.683291357421508,
+      "tokens_seen": 907673600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034077690561755246,
+      "loss": 2.7284,
+      "theoretical_loss": 3.682006127540184,
+      "tokens_seen": 910950400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000340185833175714,
+      "loss": 2.7195,
+      "theoretical_loss": 3.680726801682522,
+      "tokens_seen": 914227200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033959476073387555,
+      "loss": 2.7211,
+      "theoretical_loss": 3.679453331710889,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003390036882920371,
+      "loss": 2.7229,
+      "theoretical_loss": 3.6781856700502646,
+      "tokens_seen": 920780800
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003384126158501986,
+      "loss": 2.7281,
+      "theoretical_loss": 3.6769237696796933,
+      "tokens_seen": 924057600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003378215434083601,
+      "loss": 2.7212,
+      "theoretical_loss": 3.6756675841238913,
+      "tokens_seen": 927334400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033723047096652164,
+      "loss": 2.7393,
+      "theoretical_loss": 3.6744170674450176,
+      "tokens_seen": 930611200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033663939852468316,
+      "loss": 2.7324,
+      "theoretical_loss": 3.673172174234587,
+      "tokens_seen": 933888000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033604832608284473,
+      "loss": 2.7316,
+      "theoretical_loss": 3.6719328596055423,
+      "tokens_seen": 937164800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033545725364100625,
+      "loss": 2.7062,
+      "theoretical_loss": 3.670699079184467,
+      "tokens_seen": 940441600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003348661811991678,
+      "loss": 2.7195,
+      "theoretical_loss": 3.669470789103942,
+      "tokens_seen": 943718400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003342751087573293,
+      "loss": 2.7238,
+      "theoretical_loss": 3.6682479459950446,
+      "tokens_seen": 946995200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003336840363154908,
+      "loss": 2.6712,
+      "theoretical_loss": 3.6670305069799785,
+      "tokens_seen": 950272000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033309296387365234,
+      "loss": 2.6672,
+      "theoretical_loss": 3.6658184296648457,
+      "tokens_seen": 953548800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033250189143181386,
+      "loss": 2.6387,
+      "theoretical_loss": 3.6646116721325415,
+      "tokens_seen": 956825600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033191081898997543,
+      "loss": 2.6132,
+      "theoretical_loss": 3.6634101929357836,
+      "tokens_seen": 960102400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003313315679969737,
+      "loss": 2.6409,
+      "theoretical_loss": 3.6622139510902625,
+      "tokens_seen": 963379200
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003307404955551352,
+      "loss": 2.6332,
+      "theoretical_loss": 3.6610229060679167,
+      "tokens_seen": 966656000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033014942311329675,
+      "loss": 2.6786,
+      "theoretical_loss": 3.659837017790328,
+      "tokens_seen": 969932800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032955835067145827,
+      "loss": 2.683,
+      "theoretical_loss": 3.658656246622233,
+      "tokens_seen": 973209600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003289672782296198,
+      "loss": 2.7089,
+      "theoretical_loss": 3.6574805533651515,
+      "tokens_seen": 976486400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032837620578778136,
+      "loss": 2.6247,
+      "theoretical_loss": 3.6563098992511267,
+      "tokens_seen": 979763200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.44848857539964976,
+      "debugging/distinct-1-grams": 0.7781029350046207,
+      "debugging/distinct-2-grams": 0.9575994862285143,
+      "debugging/entropy-1-grams": 5.851953292865711,
+      "debugging/entropy-2-grams": 6.601563630419801,
+      "debugging/length": 518.3636363636364,
+      "debugging/num_segments": 11,
+      "epoch": 0.35,
+      "objective/train/docs_used": 548797,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4279284477233887,
+      "objective/train/theoretical_loss": 3.655144245936574,
+      "objective/train/tokens_used": 1003500000,
+      "theoretical_loss": 3.655144245936574,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003277851333459429,
+      "loss": 2.7072,
+      "theoretical_loss": 3.655144245936574,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003271940609041044,
+      "loss": 2.6516,
+      "theoretical_loss": 3.653983555496242,
+      "tokens_seen": 986316800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003266029884622659,
+      "loss": 2.6659,
+      "theoretical_loss": 3.6528277904172755,
+      "tokens_seen": 989593600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032601191602042745,
+      "loss": 2.705,
+      "theoretical_loss": 3.6516769135933815,
+      "tokens_seen": 992870400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032542084357858897,
+      "loss": 2.7945,
+      "theoretical_loss": 3.650530888319103,
+      "tokens_seen": 996147200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032482977113675054,
+      "loss": 2.7835,
+      "theoretical_loss": 3.649389678284182,
+      "tokens_seen": 999424000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032423869869491206,
+      "loss": 2.803,
+      "theoretical_loss": 3.6482532475680287,
+      "tokens_seen": 1002700800
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003236476262530736,
+      "loss": 2.8145,
+      "theoretical_loss": 3.6471215606342833,
+      "tokens_seen": 1005977600
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032306837526007185,
+      "loss": 2.7892,
+      "theoretical_loss": 3.645994582325468,
+      "tokens_seen": 1009254400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003224773028182334,
+      "loss": 2.7912,
+      "theoretical_loss": 3.6448722778577327,
+      "tokens_seen": 1012531200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003218862303763949,
+      "loss": 2.7827,
+      "theoretical_loss": 3.6437546128156946,
+      "tokens_seen": 1015808000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032129515793455647,
+      "loss": 2.739,
+      "theoretical_loss": 3.6426415531473566,
+      "tokens_seen": 1019084800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000320704085492718,
+      "loss": 2.7624,
+      "theoretical_loss": 3.641533065159118,
+      "tokens_seen": 1022361600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003201130130508795,
+      "loss": 2.7213,
+      "theoretical_loss": 3.6404291155108712,
+      "tokens_seen": 1025638400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031952194060904103,
+      "loss": 2.7456,
+      "theoretical_loss": 3.639329671211173,
+      "tokens_seen": 1028915200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031893086816720255,
+      "loss": 2.7553,
+      "theoretical_loss": 3.6382346996125055,
+      "tokens_seen": 1032192000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003183397957253641,
+      "loss": 2.7371,
+      "theoretical_loss": 3.6371441684066097,
+      "tokens_seen": 1035468800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003177487232835256,
+      "loss": 2.761,
+      "theoretical_loss": 3.6360580456199036,
+      "tokens_seen": 1038745600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031715765084168717,
+      "loss": 2.7745,
+      "theoretical_loss": 3.6349762996089683,
+      "tokens_seen": 1042022400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003165665783998487,
+      "loss": 2.7151,
+      "theoretical_loss": 3.633898899056115,
+      "tokens_seen": 1045299200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003159755059580102,
+      "loss": 2.7723,
+      "theoretical_loss": 3.6328258129650246,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031538443351617173,
+      "loss": 2.748,
+      "theoretical_loss": 3.6317570106564565,
+      "tokens_seen": 1051852800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031479336107433325,
+      "loss": 2.7319,
+      "theoretical_loss": 3.6306924617640295,
+      "tokens_seen": 1055129600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003142022886324948,
+      "loss": 2.7559,
+      "theoretical_loss": 3.6296321362300716,
+      "tokens_seen": 1058406400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031361121619065635,
+      "loss": 2.7457,
+      "theoretical_loss": 3.6285760043015385,
+      "tokens_seen": 1061683200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031302014374881787,
+      "loss": 2.7373,
+      "theoretical_loss": 3.6275240365259958,
+      "tokens_seen": 1064960000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003124290713069794,
+      "loss": 2.693,
+      "theoretical_loss": 3.6264762037476683,
+      "tokens_seen": 1068236800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003118379988651409,
+      "loss": 2.6899,
+      "theoretical_loss": 3.625432477103554,
+      "tokens_seen": 1071513600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031124692642330243,
+      "loss": 2.6807,
+      "theoretical_loss": 3.6243928280195976,
+      "tokens_seen": 1074790400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031065585398146395,
+      "loss": 2.6832,
+      "theoretical_loss": 3.62335722820693,
+      "tokens_seen": 1078067200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031006478153962553,
+      "loss": 2.6918,
+      "theoretical_loss": 3.6223256496581637,
+      "tokens_seen": 1081344000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030947370909778705,
+      "loss": 2.7096,
+      "theoretical_loss": 3.6212980646437485,
+      "tokens_seen": 1084620800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030888263665594857,
+      "loss": 2.6864,
+      "theoretical_loss": 3.6202744457083877,
+      "tokens_seen": 1087897600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003082915642141101,
+      "loss": 2.7229,
+      "theoretical_loss": 3.6192547656675083,
+      "tokens_seen": 1091174400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003077004917722716,
+      "loss": 2.7171,
+      "theoretical_loss": 3.618238997603788,
+      "tokens_seen": 1094451200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030710941933043313,
+      "loss": 2.7049,
+      "theoretical_loss": 3.617227114863738,
+      "tokens_seen": 1097728000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030651834688859465,
+      "loss": 2.6875,
+      "theoretical_loss": 3.6162190910543366,
+      "tokens_seen": 1101004800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030592727444675623,
+      "loss": 2.6679,
+      "theoretical_loss": 3.615214900039721,
+      "tokens_seen": 1104281600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030533620200491775,
+      "loss": 2.6854,
+      "theoretical_loss": 3.614214515937924,
+      "tokens_seen": 1107558400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030474512956307927,
+      "loss": 2.6916,
+      "theoretical_loss": 3.613217913117667,
+      "tokens_seen": 1110835200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003041540571212408,
+      "loss": 2.6977,
+      "theoretical_loss": 3.612225066195201,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003035629846794023,
+      "loss": 2.7172,
+      "theoretical_loss": 3.611235950031194,
+      "tokens_seen": 1117388800
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030297191223756383,
+      "loss": 2.7051,
+      "theoretical_loss": 3.6102505397276743,
+      "tokens_seen": 1120665600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003023808397957254,
+      "loss": 2.687,
+      "theoretical_loss": 3.60926881062501,
+      "tokens_seen": 1123942400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030178976735388693,
+      "loss": 2.6438,
+      "theoretical_loss": 3.608290738298942,
+      "tokens_seen": 1127219200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030119869491204845,
+      "loss": 2.6615,
+      "theoretical_loss": 3.6073162985576643,
+      "tokens_seen": 1130496000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030060762247020997,
+      "loss": 2.6684,
+      "theoretical_loss": 3.606345467438941,
+      "tokens_seen": 1133772800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003000165500283715,
+      "loss": 2.6597,
+      "theoretical_loss": 3.6053782212072747,
+      "tokens_seen": 1137049600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000299425477586533,
+      "loss": 2.6721,
+      "theoretical_loss": 3.604414536351113,
+      "tokens_seen": 1140326400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002988344051446946,
+      "loss": 2.6791,
+      "theoretical_loss": 3.6034543895801017,
+      "tokens_seen": 1143603200
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 634175,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4255619049072266,
+      "objective/train/theoretical_loss": 3.6024977578223742,
+      "objective/train/tokens_used": 1167340000,
+      "theoretical_loss": 3.6024977578223742,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002982433327028561,
+      "loss": 2.6508,
+      "theoretical_loss": 3.6024977578223742,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029765226026101763,
+      "loss": 2.7451,
+      "theoretical_loss": 3.6015446182218875,
+      "tokens_seen": 1150156800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029706118781917915,
+      "loss": 2.7231,
+      "theoretical_loss": 3.600594948135793,
+      "tokens_seen": 1153433600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029647011537734067,
+      "loss": 2.6868,
+      "theoretical_loss": 3.59964872513185,
+      "tokens_seen": 1156710400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029587904293550214,
+      "loss": 2.7225,
+      "theoretical_loss": 3.5987059269858763,
+      "tokens_seen": 1159987200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029528797049366366,
+      "loss": 2.7105,
+      "theoretical_loss": 3.5977665316792375,
+      "tokens_seen": 1163264000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029469689805182523,
+      "loss": 2.6838,
+      "theoretical_loss": 3.5968305173963744,
+      "tokens_seen": 1166540800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029410582560998675,
+      "loss": 2.6623,
+      "theoretical_loss": 3.5958978625223628,
+      "tokens_seen": 1169817600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002935147531681483,
+      "loss": 2.6717,
+      "theoretical_loss": 3.5949685456405165,
+      "tokens_seen": 1173094400
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002929236807263098,
+      "loss": 2.6992,
+      "theoretical_loss": 3.5940425455300176,
+      "tokens_seen": 1176371200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002923326082844713,
+      "loss": 2.6954,
+      "theoretical_loss": 3.593119841163589,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029174153584263284,
+      "loss": 2.6819,
+      "theoretical_loss": 3.5922004117051944,
+      "tokens_seen": 1182924800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002911504634007944,
+      "loss": 2.6886,
+      "theoretical_loss": 3.5912842365077777,
+      "tokens_seen": 1186201600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029055939095895593,
+      "loss": 2.6893,
+      "theoretical_loss": 3.5903712951110305,
+      "tokens_seen": 1189478400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028996831851711745,
+      "loss": 2.6813,
+      "theoretical_loss": 3.5894615672391947,
+      "tokens_seen": 1192755200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000289377246075279,
+      "loss": 2.7106,
+      "theoretical_loss": 3.5885550327988973,
+      "tokens_seen": 1196032000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002887861736334405,
+      "loss": 2.6777,
+      "theoretical_loss": 3.587651671877014,
+      "tokens_seen": 1199308800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000288195101191602,
+      "loss": 2.6959,
+      "theoretical_loss": 3.5867514647385663,
+      "tokens_seen": 1202585600
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002876040287497636,
+      "loss": 2.6972,
+      "theoretical_loss": 3.585854391824647,
+      "tokens_seen": 1205862400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002870129563079251,
+      "loss": 2.6602,
+      "theoretical_loss": 3.584960433750375,
+      "tokens_seen": 1209139200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028642188386608663,
+      "loss": 2.6846,
+      "theoretical_loss": 3.5840695713028827,
+      "tokens_seen": 1212416000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028583081142424815,
+      "loss": 2.6813,
+      "theoretical_loss": 3.5831817854393266,
+      "tokens_seen": 1215692800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028523973898240967,
+      "loss": 2.6898,
+      "theoretical_loss": 3.582297057284933,
+      "tokens_seen": 1218969600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002846486665405712,
+      "loss": 2.7131,
+      "theoretical_loss": 3.5814153681310623,
+      "tokens_seen": 1222246400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002840575940987327,
+      "loss": 2.7068,
+      "theoretical_loss": 3.5805366994333125,
+      "tokens_seen": 1225523200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002834665216568943,
+      "loss": 2.6921,
+      "theoretical_loss": 3.5796610328096365,
+      "tokens_seen": 1228800000
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002828754492150558,
+      "loss": 2.6975,
+      "theoretical_loss": 3.578788350038497,
+      "tokens_seen": 1232076800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028228437677321733,
+      "loss": 2.6815,
+      "theoretical_loss": 3.5779186330570405,
+      "tokens_seen": 1235353600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028169330433137885,
+      "loss": 2.6759,
+      "theoretical_loss": 3.5770518639592983,
+      "tokens_seen": 1238630400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028110223188954037,
+      "loss": 2.7393,
+      "theoretical_loss": 3.5761880249944147,
+      "tokens_seen": 1241907200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002805111594477019,
+      "loss": 2.7223,
+      "theoretical_loss": 3.5753270985648973,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027992008700586347,
+      "loss": 2.716,
+      "theoretical_loss": 3.574469067224892,
+      "tokens_seen": 1248460800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.000279329014564025,
+      "loss": 2.7334,
+      "theoretical_loss": 3.573613913678484,
+      "tokens_seen": 1251737600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002787379421221865,
+      "loss": 2.7213,
+      "theoretical_loss": 3.57276162077802,
+      "tokens_seen": 1255014400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027814686968034803,
+      "loss": 2.712,
+      "theoretical_loss": 3.5719121715224524,
+      "tokens_seen": 1258291200
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027755579723850955,
+      "loss": 2.6879,
+      "theoretical_loss": 3.571065549055712,
+      "tokens_seen": 1261568000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027696472479667107,
+      "loss": 2.6854,
+      "theoretical_loss": 3.5702217366650935,
+      "tokens_seen": 1264844800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027637365235483265,
+      "loss": 2.6818,
+      "theoretical_loss": 3.5693807177796737,
+      "tokens_seen": 1268121600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027578257991299417,
+      "loss": 2.6228,
+      "theoretical_loss": 3.5685424759687434,
+      "tokens_seen": 1271398400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002751915074711557,
+      "loss": 2.6779,
+      "theoretical_loss": 3.567706994940263,
+      "tokens_seen": 1274675200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002746004350293172,
+      "loss": 2.6876,
+      "theoretical_loss": 3.5668742585393405,
+      "tokens_seen": 1277952000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027400936258747873,
+      "loss": 2.6531,
+      "theoretical_loss": 3.566044250746728,
+      "tokens_seen": 1281228800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027341829014564025,
+      "loss": 2.6545,
+      "theoretical_loss": 3.5652169556773403,
+      "tokens_seen": 1284505600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002728272177038018,
+      "loss": 2.6209,
+      "theoretical_loss": 3.5643923575787912,
+      "tokens_seen": 1287782400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027223614526196335,
+      "loss": 2.6529,
+      "theoretical_loss": 3.563570440829951,
+      "tokens_seen": 1291059200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027164507282012487,
+      "loss": 2.6148,
+      "theoretical_loss": 3.562751189939524,
+      "tokens_seen": 1294336000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002710540003782864,
+      "loss": 2.6728,
+      "theoretical_loss": 3.5619345895446424,
+      "tokens_seen": 1297612800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002704629279364479,
+      "loss": 2.6349,
+      "theoretical_loss": 3.561120624409482,
+      "tokens_seen": 1300889600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026987185549460943,
+      "loss": 2.6384,
+      "theoretical_loss": 3.560309279423894,
+      "tokens_seen": 1304166400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026928078305277095,
+      "loss": 2.6076,
+      "theoretical_loss": 3.5595005396020554,
+      "tokens_seen": 1307443200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5082619340855203,
+      "debugging/distinct-1-grams": 0.7794681642048736,
+      "debugging/distinct-2-grams": 0.9351864119489364,
+      "debugging/entropy-1-grams": 6.068338078693592,
+      "debugging/entropy-2-grams": 6.8266304879748,
+      "debugging/length": 505.06666666666666,
+      "debugging/num_segments": 15,
+      "epoch": 0.47,
+      "objective/train/docs_used": 722411,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4245951175689697,
+      "objective/train/theoretical_loss": 3.558694390081137,
+      "objective/train/tokens_used": 1331180000,
+      "theoretical_loss": 3.558694390081137,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002686897106109325,
+      "loss": 2.6127,
+      "theoretical_loss": 3.558694390081137,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026809863816909405,
+      "loss": 2.5874,
+      "theoretical_loss": 3.5578908161199934,
+      "tokens_seen": 1313996800
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026750756572725557,
+      "loss": 2.6153,
+      "theoretical_loss": 3.5570898030978584,
+      "tokens_seen": 1317273600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002669164932854171,
+      "loss": 2.6265,
+      "theoretical_loss": 3.556291336513074,
+      "tokens_seen": 1320550400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026632542084357855,
+      "loss": 2.6162,
+      "theoretical_loss": 3.5554954019818235,
+      "tokens_seen": 1323827200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002657343484017401,
+      "loss": 2.612,
+      "theoretical_loss": 3.554701985236883,
+      "tokens_seen": 1327104000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026514327595990165,
+      "loss": 2.6526,
+      "theoretical_loss": 3.553911072126394,
+      "tokens_seen": 1330380800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026455220351806317,
+      "loss": 2.6474,
+      "theoretical_loss": 3.5531226486126504,
+      "tokens_seen": 1333657600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002639611310762247,
+      "loss": 2.6462,
+      "theoretical_loss": 3.552336700770896,
+      "tokens_seen": 1336934400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002633700586343862,
+      "loss": 2.6368,
+      "theoretical_loss": 3.5515532147881443,
+      "tokens_seen": 1340211200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026277898619254773,
+      "loss": 2.6594,
+      "theoretical_loss": 3.5507721769620098,
+      "tokens_seen": 1343488000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026218791375070925,
+      "loss": 2.6582,
+      "theoretical_loss": 3.549993573699556,
+      "tokens_seen": 1346764800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026159684130887083,
+      "loss": 2.6446,
+      "theoretical_loss": 3.5492173915161565,
+      "tokens_seen": 1350041600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026100576886703235,
+      "loss": 2.6281,
+      "theoretical_loss": 3.548443617034371,
+      "tokens_seen": 1353318400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026041469642519387,
+      "loss": 2.6538,
+      "theoretical_loss": 3.547672236982839,
+      "tokens_seen": 1356595200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002598236239833554,
+      "loss": 2.6518,
+      "theoretical_loss": 3.5469032381951804,
+      "tokens_seen": 1359872000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002592325515415169,
+      "loss": 2.6576,
+      "theoretical_loss": 3.5461366076089202,
+      "tokens_seen": 1363148800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025864147909967843,
+      "loss": 2.6604,
+      "theoretical_loss": 3.5453723322644146,
+      "tokens_seen": 1366425600
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025805040665783995,
+      "loss": 2.6435,
+      "theoretical_loss": 3.544610399303803,
+      "tokens_seen": 1369702400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025745933421600153,
+      "loss": 2.6422,
+      "theoretical_loss": 3.5438507959699637,
+      "tokens_seen": 1372979200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025686826177416305,
+      "loss": 2.6572,
+      "theoretical_loss": 3.5430935096054883,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025627718933232457,
+      "loss": 2.6133,
+      "theoretical_loss": 3.5423385276516663,
+      "tokens_seen": 1379532800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002556861168904861,
+      "loss": 2.6267,
+      "theoretical_loss": 3.5415858376474825,
+      "tokens_seen": 1382809600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002550950444486476,
+      "loss": 2.5951,
+      "theoretical_loss": 3.5408354272286298,
+      "tokens_seen": 1386086400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025450397200680913,
+      "loss": 2.6355,
+      "theoretical_loss": 3.540087284126531,
+      "tokens_seen": 1389363200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002539128995649707,
+      "loss": 2.6396,
+      "theoretical_loss": 3.539341396167372,
+      "tokens_seen": 1392640000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025332182712313223,
+      "loss": 2.6724,
+      "theoretical_loss": 3.538597751271153,
+      "tokens_seen": 1395916800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025273075468129375,
+      "loss": 2.6724,
+      "theoretical_loss": 3.5378563374507443,
+      "tokens_seen": 1399193600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025213968223945527,
+      "loss": 2.6523,
+      "theoretical_loss": 3.5371171428109576,
+      "tokens_seen": 1402470400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002515486097976168,
+      "loss": 2.6513,
+      "theoretical_loss": 3.5363801555476275,
+      "tokens_seen": 1405747200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025096935880461506,
+      "loss": 2.6682,
+      "theoretical_loss": 3.5356453639467063,
+      "tokens_seen": 1409024000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025037828636277664,
+      "loss": 2.6703,
+      "theoretical_loss": 3.534912756383366,
+      "tokens_seen": 1412300800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024978721392093816,
+      "loss": 2.6515,
+      "theoretical_loss": 3.534182321321116,
+      "tokens_seen": 1415577600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002491961414790997,
+      "loss": 2.6405,
+      "theoretical_loss": 3.5334540473109275,
+      "tokens_seen": 1418854400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002486050690372612,
+      "loss": 2.6204,
+      "theoretical_loss": 3.532727922990372,
+      "tokens_seen": 1422131200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002480139965954227,
+      "loss": 2.6124,
+      "theoretical_loss": 3.532003937082766,
+      "tokens_seen": 1425408000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002474229241535843,
+      "loss": 2.651,
+      "theoretical_loss": 3.5312820783963303,
+      "tokens_seen": 1428684800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002468318517117458,
+      "loss": 2.6744,
+      "theoretical_loss": 3.5305623358233578,
+      "tokens_seen": 1431961600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024624077926990734,
+      "loss": 2.6549,
+      "theoretical_loss": 3.529844698339391,
+      "tokens_seen": 1435238400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024564970682806886,
+      "loss": 2.6189,
+      "theoretical_loss": 3.5291291550024093,
+      "tokens_seen": 1438515200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002450586343862304,
+      "loss": 2.6349,
+      "theoretical_loss": 3.5284156949520264,
+      "tokens_seen": 1441792000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002444675619443919,
+      "loss": 2.6156,
+      "theoretical_loss": 3.5277043074086993,
+      "tokens_seen": 1445068800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024387648950255345,
+      "loss": 2.5742,
+      "theoretical_loss": 3.5269949816729422,
+      "tokens_seen": 1448345600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024328541706071497,
+      "loss": 2.6289,
+      "theoretical_loss": 3.5262877071245526,
+      "tokens_seen": 1451622400
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002426943446188765,
+      "loss": 2.6272,
+      "theoretical_loss": 3.525582473221849,
+      "tokens_seen": 1454899200
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024210327217703804,
+      "loss": 2.6192,
+      "theoretical_loss": 3.5248792695009126,
+      "tokens_seen": 1458176000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024151219973519956,
+      "loss": 2.6341,
+      "theoretical_loss": 3.5241780855748415,
+      "tokens_seen": 1461452800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024092112729336108,
+      "loss": 2.6014,
+      "theoretical_loss": 3.523478911133012,
+      "tokens_seen": 1464729600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002403300548515226,
+      "loss": 2.6057,
+      "theoretical_loss": 3.5227817359403506,
+      "tokens_seen": 1468006400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023973898240968415,
+      "loss": 2.6116,
+      "theoretical_loss": 3.5220865498366107,
+      "tokens_seen": 1471283200
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 814199,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.9345042705535889,
+      "objective/train/theoretical_loss": 3.5213933427356636,
+      "objective/train/tokens_used": 1495020000,
+      "theoretical_loss": 3.5213933427356636,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023914790996784567,
+      "loss": 2.5468,
+      "theoretical_loss": 3.5213933427356636,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002385568375260072,
+      "loss": 2.572,
+      "theoretical_loss": 3.5207021046247924,
+      "tokens_seen": 1477836800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023796576508416874,
+      "loss": 2.5631,
+      "theoretical_loss": 3.520012825563997,
+      "tokens_seen": 1481113600
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023737469264233026,
+      "loss": 2.559,
+      "theoretical_loss": 3.519325495685306,
+      "tokens_seen": 1484390400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023678362020049178,
+      "loss": 2.5593,
+      "theoretical_loss": 3.5186401051920977,
+      "tokens_seen": 1487667200
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023619254775865333,
+      "loss": 2.5695,
+      "theoretical_loss": 3.517956644358428,
+      "tokens_seen": 1490944000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023560147531681482,
+      "loss": 2.5973,
+      "theoretical_loss": 3.517275103528368,
+      "tokens_seen": 1494220800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023501040287497634,
+      "loss": 2.5707,
+      "theoretical_loss": 3.516595473115344,
+      "tokens_seen": 1497497600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002344193304331379,
+      "loss": 2.5879,
+      "theoretical_loss": 3.5159177436014932,
+      "tokens_seen": 1500774400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002338282579912994,
+      "loss": 2.6169,
+      "theoretical_loss": 3.5152419055370205,
+      "tokens_seen": 1504051200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023323718554946093,
+      "loss": 2.6021,
+      "theoretical_loss": 3.5145679495395643,
+      "tokens_seen": 1507328000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023264611310762248,
+      "loss": 2.6079,
+      "theoretical_loss": 3.513895866293571,
+      "tokens_seen": 1510604800
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000232055040665784,
+      "loss": 2.6083,
+      "theoretical_loss": 3.5132256465496736,
+      "tokens_seen": 1513881600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023146396822394552,
+      "loss": 2.6102,
+      "theoretical_loss": 3.5125572811240833,
+      "tokens_seen": 1517158400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023088471723094385,
+      "loss": 2.6205,
+      "theoretical_loss": 3.5118907608979786,
+      "tokens_seen": 1520435200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023029364478910537,
+      "loss": 2.6159,
+      "theoretical_loss": 3.5112260768169135,
+      "tokens_seen": 1523712000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002297025723472669,
+      "loss": 2.6385,
+      "theoretical_loss": 3.510563219890219,
+      "tokens_seen": 1526988800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022911149990542844,
+      "loss": 2.6275,
+      "theoretical_loss": 3.509902181190423,
+      "tokens_seen": 1530265600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022852042746358996,
+      "loss": 2.653,
+      "theoretical_loss": 3.50924295185267,
+      "tokens_seen": 1533542400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022792935502175148,
+      "loss": 2.6651,
+      "theoretical_loss": 3.508585523074149,
+      "tokens_seen": 1536819200
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.000227338282579913,
+      "loss": 2.6474,
+      "theoretical_loss": 3.5079298861135295,
+      "tokens_seen": 1540096000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022674721013807455,
+      "loss": 2.6334,
+      "theoretical_loss": 3.5072760322904006,
+      "tokens_seen": 1543372800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022615613769623604,
+      "loss": 2.6271,
+      "theoretical_loss": 3.50662395298472,
+      "tokens_seen": 1546649600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022556506525439756,
+      "loss": 2.6589,
+      "theoretical_loss": 3.505973639636264,
+      "tokens_seen": 1549926400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002249739928125591,
+      "loss": 2.6542,
+      "theoretical_loss": 3.5053250837440926,
+      "tokens_seen": 1553203200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022438292037072063,
+      "loss": 2.6032,
+      "theoretical_loss": 3.5046782768660085,
+      "tokens_seen": 1556480000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022379184792888215,
+      "loss": 2.6175,
+      "theoretical_loss": 3.5040332106180347,
+      "tokens_seen": 1559756800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002232007754870437,
+      "loss": 2.5869,
+      "theoretical_loss": 3.503389876673886,
+      "tokens_seen": 1563033600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022260970304520522,
+      "loss": 2.6156,
+      "theoretical_loss": 3.5027482667644554,
+      "tokens_seen": 1566310400
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022201863060336674,
+      "loss": 2.5848,
+      "theoretical_loss": 3.502108372677303,
+      "tokens_seen": 1569587200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002214275581615283,
+      "loss": 2.6439,
+      "theoretical_loss": 3.5014701862561486,
+      "tokens_seen": 1572864000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002208364857196898,
+      "loss": 2.5951,
+      "theoretical_loss": 3.5008336994003724,
+      "tokens_seen": 1576140800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022024541327785133,
+      "loss": 2.5775,
+      "theoretical_loss": 3.500198904064521,
+      "tokens_seen": 1579417600
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021965434083601288,
+      "loss": 2.6073,
+      "theoretical_loss": 3.499565792257817,
+      "tokens_seen": 1582694400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002190632683941744,
+      "loss": 2.6147,
+      "theoretical_loss": 3.4989343560436756,
+      "tokens_seen": 1585971200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021847219595233592,
+      "loss": 2.6361,
+      "theoretical_loss": 3.498304587539227,
+      "tokens_seen": 1589248000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021788112351049747,
+      "loss": 2.6307,
+      "theoretical_loss": 3.4976764789148422,
+      "tokens_seen": 1592524800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.000217290051068659,
+      "loss": 2.6112,
+      "theoretical_loss": 3.4970500223936627,
+      "tokens_seen": 1595801600
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002166989786268205,
+      "loss": 2.5938,
+      "theoretical_loss": 3.496425210251142,
+      "tokens_seen": 1599078400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021610790618498203,
+      "loss": 2.584,
+      "theoretical_loss": 3.4958020348145835,
+      "tokens_seen": 1602355200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021551683374314358,
+      "loss": 2.6048,
+      "theoretical_loss": 3.4951804884626885,
+      "tokens_seen": 1605632000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002149257613013051,
+      "loss": 2.5646,
+      "theoretical_loss": 3.494560563625107,
+      "tokens_seen": 1608908800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021433468885946662,
+      "loss": 2.6155,
+      "theoretical_loss": 3.4939422527819968,
+      "tokens_seen": 1612185600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021374361641762817,
+      "loss": 2.6374,
+      "theoretical_loss": 3.4933255484635826,
+      "tokens_seen": 1615462400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002131525439757897,
+      "loss": 2.6339,
+      "theoretical_loss": 3.4927104432497202,
+      "tokens_seen": 1618739200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002125614715339512,
+      "loss": 2.6319,
+      "theoretical_loss": 3.4920969297694713,
+      "tokens_seen": 1622016000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021197039909211273,
+      "loss": 2.5745,
+      "theoretical_loss": 3.491485000700675,
+      "tokens_seen": 1625292800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021137932665027425,
+      "loss": 2.568,
+      "theoretical_loss": 3.4908746487695304,
+      "tokens_seen": 1628569600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021078825420843577,
+      "loss": 2.5776,
+      "theoretical_loss": 3.4902658667501774,
+      "tokens_seen": 1631846400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021019718176659732,
+      "loss": 2.5956,
+      "theoretical_loss": 3.489658647464288,
+      "tokens_seen": 1635123200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6066558165014577,
+      "debugging/distinct-1-grams": 0.7744435380720751,
+      "debugging/distinct-2-grams": 0.9588156094739437,
+      "debugging/entropy-1-grams": 6.213223194067869,
+      "debugging/entropy-2-grams": 7.226898787830292,
+      "debugging/length": 539.8571428571429,
+      "debugging/num_segments": 21,
+      "epoch": 0.59,
+      "objective/train/docs_used": 904135,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.1725375652313232,
+      "objective/train/theoretical_loss": 3.489052983780656,
+      "objective/train/tokens_used": 1658860000,
+      "theoretical_loss": 3.489052983780656,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020960610932475884,
+      "loss": 2.6059,
+      "theoretical_loss": 3.489052983780656,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020901503688292036,
+      "loss": 2.5869,
+      "theoretical_loss": 3.488448868614799,
+      "tokens_seen": 1641676800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002084239644410819,
+      "loss": 2.5712,
+      "theoretical_loss": 3.4878462949285556,
+      "tokens_seen": 1644953600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020783289199924343,
+      "loss": 2.5823,
+      "theoretical_loss": 3.4872452557296922,
+      "tokens_seen": 1648230400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020724181955740495,
+      "loss": 2.5971,
+      "theoretical_loss": 3.486645744071512,
+      "tokens_seen": 1651507200
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002066507471155665,
+      "loss": 2.617,
+      "theoretical_loss": 3.486047753052471,
+      "tokens_seen": 1654784000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020605967467372802,
+      "loss": 2.6075,
+      "theoretical_loss": 3.4854512758157945,
+      "tokens_seen": 1658060800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020546860223188954,
+      "loss": 2.6014,
+      "theoretical_loss": 3.484856305549097,
+      "tokens_seen": 1661337600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020487752979005106,
+      "loss": 2.6134,
+      "theoretical_loss": 3.484262835484011,
+      "tokens_seen": 1664614400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002042864573482126,
+      "loss": 2.594,
+      "theoretical_loss": 3.483670858895815,
+      "tokens_seen": 1667891200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020369538490637413,
+      "loss": 2.5496,
+      "theoretical_loss": 3.4830803691030683,
+      "tokens_seen": 1671168000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020311613391337243,
+      "loss": 2.553,
+      "theoretical_loss": 3.4824913594672458,
+      "tokens_seen": 1674444800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002025487043692075,
+      "loss": 2.5672,
+      "theoretical_loss": 3.481903823392381,
+      "tokens_seen": 1677721600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020195763192736903,
+      "loss": 2.5968,
+      "theoretical_loss": 3.4813177543247096,
+      "tokens_seen": 1680998400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020136655948553055,
+      "loss": 2.5675,
+      "theoretical_loss": 3.480733145752319,
+      "tokens_seen": 1684275200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002007754870436921,
+      "loss": 2.5424,
+      "theoretical_loss": 3.4801499912048,
+      "tokens_seen": 1687552000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020018441460185361,
+      "loss": 2.6235,
+      "theoretical_loss": 3.4795682842529,
+      "tokens_seen": 1690828800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019959334216001514,
+      "loss": 2.6032,
+      "theoretical_loss": 3.4789880185081863,
+      "tokens_seen": 1694105600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019900226971817668,
+      "loss": 2.5724,
+      "theoretical_loss": 3.4784091876227046,
+      "tokens_seen": 1697382400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019841119727633818,
+      "loss": 2.5533,
+      "theoretical_loss": 3.4778317852886476,
+      "tokens_seen": 1700659200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001978201248344997,
+      "loss": 2.5466,
+      "theoretical_loss": 3.4772558052380216,
+      "tokens_seen": 1703936000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019722905239266125,
+      "loss": 2.5389,
+      "theoretical_loss": 3.476681241242323,
+      "tokens_seen": 1707212800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019663797995082277,
+      "loss": 2.5519,
+      "theoretical_loss": 3.4761080871122116,
+      "tokens_seen": 1710489600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001960469075089843,
+      "loss": 2.5713,
+      "theoretical_loss": 3.4755363366971896,
+      "tokens_seen": 1713766400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019545583506714584,
+      "loss": 2.5817,
+      "theoretical_loss": 3.4749659838852858,
+      "tokens_seen": 1717043200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019486476262530736,
+      "loss": 2.5921,
+      "theoretical_loss": 3.4743970226027403,
+      "tokens_seen": 1720320000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019427369018346888,
+      "loss": 2.5569,
+      "theoretical_loss": 3.4738294468136934,
+      "tokens_seen": 1723596800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019368261774163042,
+      "loss": 2.5986,
+      "theoretical_loss": 3.473263250519878,
+      "tokens_seen": 1726873600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019309154529979195,
+      "loss": 2.5739,
+      "theoretical_loss": 3.4726984277603146,
+      "tokens_seen": 1730150400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019250047285795347,
+      "loss": 2.6191,
+      "theoretical_loss": 3.472134972611009,
+      "tokens_seen": 1733427200
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019190940041611501,
+      "loss": 2.5877,
+      "theoretical_loss": 3.471572879184653,
+      "tokens_seen": 1736704000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019131832797427654,
+      "loss": 2.6027,
+      "theoretical_loss": 3.4710121416303306,
+      "tokens_seen": 1739980800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019072725553243806,
+      "loss": 2.6162,
+      "theoretical_loss": 3.4704527541332224,
+      "tokens_seen": 1743257600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001901361830905996,
+      "loss": 2.6134,
+      "theoretical_loss": 3.469894710914317,
+      "tokens_seen": 1746534400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018954511064876112,
+      "loss": 2.607,
+      "theoretical_loss": 3.469338006230123,
+      "tokens_seen": 1749811200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018895403820692265,
+      "loss": 2.6137,
+      "theoretical_loss": 3.468782634372387,
+      "tokens_seen": 1753088000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018836296576508417,
+      "loss": 2.5984,
+      "theoretical_loss": 3.4682285896678087,
+      "tokens_seen": 1756364800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018777189332324571,
+      "loss": 2.5816,
+      "theoretical_loss": 3.467675866477763,
+      "tokens_seen": 1759641600
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018718082088140723,
+      "loss": 2.5883,
+      "theoretical_loss": 3.4671244591980264,
+      "tokens_seen": 1762918400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018658974843956876,
+      "loss": 2.5682,
+      "theoretical_loss": 3.466574362258501,
+      "tokens_seen": 1766195200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001859986759977303,
+      "loss": 2.6085,
+      "theoretical_loss": 3.4660255701229445,
+      "tokens_seen": 1769472000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018540760355589182,
+      "loss": 2.5767,
+      "theoretical_loss": 3.4654780772887026,
+      "tokens_seen": 1772748800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018481653111405335,
+      "loss": 2.6526,
+      "theoretical_loss": 3.464931878286443,
+      "tokens_seen": 1776025600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001842254586722149,
+      "loss": 2.6031,
+      "theoretical_loss": 3.4643869676798937,
+      "tokens_seen": 1779302400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001836343862303764,
+      "loss": 2.6302,
+      "theoretical_loss": 3.463843340065581,
+      "tokens_seen": 1782579200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001830433137885379,
+      "loss": 2.6195,
+      "theoretical_loss": 3.4633009900725735,
+      "tokens_seen": 1785856000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018245224134669946,
+      "loss": 2.6156,
+      "theoretical_loss": 3.4627599123622255,
+      "tokens_seen": 1789132800
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018186116890486098,
+      "loss": 2.6249,
+      "theoretical_loss": 3.462220101627925,
+      "tokens_seen": 1792409600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001812700964630225,
+      "loss": 2.6304,
+      "theoretical_loss": 3.4616815525948432,
+      "tokens_seen": 1795686400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018067902402118404,
+      "loss": 2.6226,
+      "theoretical_loss": 3.4611442600196867,
+      "tokens_seen": 1798963200
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 992676,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.774245262145996,
+      "objective/train/theoretical_loss": 3.460608218690451,
+      "objective/train/tokens_used": 1822700000,
+      "theoretical_loss": 3.460608218690451,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018008795157934557,
+      "loss": 2.6267,
+      "theoretical_loss": 3.460608218690451,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001794968791375071,
+      "loss": 2.6362,
+      "theoretical_loss": 3.460073423426179,
+      "tokens_seen": 1805516800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017890580669566863,
+      "loss": 2.59,
+      "theoretical_loss": 3.4595398690767176,
+      "tokens_seen": 1808793600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017831473425383016,
+      "loss": 2.5862,
+      "theoretical_loss": 3.459007550522482,
+      "tokens_seen": 1812070400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017772366181199168,
+      "loss": 2.6077,
+      "theoretical_loss": 3.4584764626742173,
+      "tokens_seen": 1815347200
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001771325893701532,
+      "loss": 2.6204,
+      "theoretical_loss": 3.4579466004727664,
+      "tokens_seen": 1818624000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017654151692831474,
+      "loss": 2.5894,
+      "theoretical_loss": 3.4574179588888345,
+      "tokens_seen": 1821900800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017595044448647627,
+      "loss": 2.5915,
+      "theoretical_loss": 3.456890532922765,
+      "tokens_seen": 1825177600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017535937204463779,
+      "loss": 2.5802,
+      "theoretical_loss": 3.456364317604306,
+      "tokens_seen": 1828454400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017476829960279933,
+      "loss": 2.5947,
+      "theoretical_loss": 3.455839307992391,
+      "tokens_seen": 1831731200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017417722716096085,
+      "loss": 2.5696,
+      "theoretical_loss": 3.45531549917491,
+      "tokens_seen": 1835008000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017358615471912238,
+      "loss": 2.5919,
+      "theoretical_loss": 3.454792886268492,
+      "tokens_seen": 1838284800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017299508227728392,
+      "loss": 2.5618,
+      "theoretical_loss": 3.454271464418285,
+      "tokens_seen": 1841561600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017240400983544544,
+      "loss": 2.5619,
+      "theoretical_loss": 3.4537512287977403,
+      "tokens_seen": 1844838400
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017181293739360697,
+      "loss": 2.5485,
+      "theoretical_loss": 3.4532321746083956,
+      "tokens_seen": 1848115200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001712218649517685,
+      "loss": 2.5733,
+      "theoretical_loss": 3.4527142970796625,
+      "tokens_seen": 1851392000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017063079250993003,
+      "loss": 2.5721,
+      "theoretical_loss": 3.4521975914686163,
+      "tokens_seen": 1854668800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017003972006809155,
+      "loss": 2.548,
+      "theoretical_loss": 3.4516820530597867,
+      "tokens_seen": 1857945600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016944864762625308,
+      "loss": 2.5463,
+      "theoretical_loss": 3.4511676771649524,
+      "tokens_seen": 1861222400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001688575751844146,
+      "loss": 2.5482,
+      "theoretical_loss": 3.450654459122932,
+      "tokens_seen": 1864499200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016826650274257612,
+      "loss": 2.5628,
+      "theoretical_loss": 3.4501423942993843,
+      "tokens_seen": 1867776000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016767543030073766,
+      "loss": 2.5553,
+      "theoretical_loss": 3.449631478086607,
+      "tokens_seen": 1871052800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016708435785889919,
+      "loss": 2.5791,
+      "theoretical_loss": 3.449121705903335,
+      "tokens_seen": 1874329600
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001664932854170607,
+      "loss": 2.5414,
+      "theoretical_loss": 3.448613073194544,
+      "tokens_seen": 1877606400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016590221297522223,
+      "loss": 2.5739,
+      "theoretical_loss": 3.4481055754312564,
+      "tokens_seen": 1880883200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016531114053338378,
+      "loss": 2.5806,
+      "theoretical_loss": 3.4475992081103435,
+      "tokens_seen": 1884160000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001647200680915453,
+      "loss": 2.579,
+      "theoretical_loss": 3.4470939667543377,
+      "tokens_seen": 1887436800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016412899564970682,
+      "loss": 2.5692,
+      "theoretical_loss": 3.4465898469112366,
+      "tokens_seen": 1890713600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016353792320786836,
+      "loss": 2.5952,
+      "theoretical_loss": 3.4460868441543218,
+      "tokens_seen": 1893990400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016294685076602989,
+      "loss": 2.5986,
+      "theoretical_loss": 3.445584954081964,
+      "tokens_seen": 1897267200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001623557783241914,
+      "loss": 2.6265,
+      "theoretical_loss": 3.4450841723174426,
+      "tokens_seen": 1900544000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016176470588235295,
+      "loss": 2.618,
+      "theoretical_loss": 3.444584494508761,
+      "tokens_seen": 1903820800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016117363344051447,
+      "loss": 2.6097,
+      "theoretical_loss": 3.444085916328465,
+      "tokens_seen": 1907097600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.000160582560998676,
+      "loss": 2.6564,
+      "theoretical_loss": 3.443588433473461,
+      "tokens_seen": 1910374400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015999148855683754,
+      "loss": 2.5904,
+      "theoretical_loss": 3.4430920416648396,
+      "tokens_seen": 1913651200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015940041611499906,
+      "loss": 2.6219,
+      "theoretical_loss": 3.4425967366476957,
+      "tokens_seen": 1916928000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015880934367316059,
+      "loss": 2.5898,
+      "theoretical_loss": 3.442102514190956,
+      "tokens_seen": 1920204800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015821827123132213,
+      "loss": 2.5402,
+      "theoretical_loss": 3.4416093700872032,
+      "tokens_seen": 1923481600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015762719878948365,
+      "loss": 2.5642,
+      "theoretical_loss": 3.441117300152505,
+      "tokens_seen": 1926758400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015703612634764517,
+      "loss": 2.5187,
+      "theoretical_loss": 3.440626300226242,
+      "tokens_seen": 1930035200
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015644505390580672,
+      "loss": 2.5132,
+      "theoretical_loss": 3.4401363661709397,
+      "tokens_seen": 1933312000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015585398146396824,
+      "loss": 2.5207,
+      "theoretical_loss": 3.4396474938720996,
+      "tokens_seen": 1936588800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015526290902212974,
+      "loss": 2.51,
+      "theoretical_loss": 3.439159679238035,
+      "tokens_seen": 1939865600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015467183658029128,
+      "loss": 2.5624,
+      "theoretical_loss": 3.4386729181997024,
+      "tokens_seen": 1943142400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001540807641384528,
+      "loss": 2.5561,
+      "theoretical_loss": 3.438187206710543,
+      "tokens_seen": 1946419200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015348969169661433,
+      "loss": 2.5817,
+      "theoretical_loss": 3.437702540746318,
+      "tokens_seen": 1949696000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015289861925477585,
+      "loss": 2.5427,
+      "theoretical_loss": 3.437218916304949,
+      "tokens_seen": 1952972800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001523075468129374,
+      "loss": 2.5414,
+      "theoretical_loss": 3.4367363294063575,
+      "tokens_seen": 1956249600
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015171647437109892,
+      "loss": 2.5211,
+      "theoretical_loss": 3.4362547760923103,
+      "tokens_seen": 1959526400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015112540192926044,
+      "loss": 2.5001,
+      "theoretical_loss": 3.4357742524262616,
+      "tokens_seen": 1962803200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5371944890135606,
+      "debugging/distinct-1-grams": 0.7829191065508077,
+      "debugging/distinct-2-grams": 0.9613208553640454,
+      "debugging/entropy-1-grams": 6.129464033853804,
+      "debugging/entropy-2-grams": 7.092336863194735,
+      "debugging/length": 486.5,
+      "debugging/num_segments": 20,
+      "epoch": 0.7,
+      "objective/train/docs_used": 1082644,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.513176918029785,
+      "objective/train/theoretical_loss": 3.4352947544931967,
+      "objective/train/tokens_used": 1986540000,
+      "theoretical_loss": 3.4352947544931967,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015053432948742198,
+      "loss": 2.5002,
+      "theoretical_loss": 3.4352947544931967,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001499432570455835,
+      "loss": 2.5539,
+      "theoretical_loss": 3.4348162783994827,
+      "tokens_seen": 1969356800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014935218460374503,
+      "loss": 2.5753,
+      "theoretical_loss": 3.4343388202727114,
+      "tokens_seen": 1972633600
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014876111216190657,
+      "loss": 2.6063,
+      "theoretical_loss": 3.4338623762615526,
+      "tokens_seen": 1975910400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001481700397200681,
+      "loss": 2.6009,
+      "theoretical_loss": 3.4333869425356025,
+      "tokens_seen": 1979187200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014757896727822962,
+      "loss": 2.6163,
+      "theoretical_loss": 3.4329125152852367,
+      "tokens_seen": 1982464000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014698789483639116,
+      "loss": 2.6337,
+      "theoretical_loss": 3.4324390907214624,
+      "tokens_seen": 1985740800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014639682239455268,
+      "loss": 2.5941,
+      "theoretical_loss": 3.431966665075773,
+      "tokens_seen": 1989017600
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001458057499527142,
+      "loss": 2.6072,
+      "theoretical_loss": 3.4314952346000047,
+      "tokens_seen": 1992294400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014521467751087575,
+      "loss": 2.5971,
+      "theoretical_loss": 3.4310247955661923,
+      "tokens_seen": 1995571200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014462360506903727,
+      "loss": 2.5698,
+      "theoretical_loss": 3.430555344266428,
+      "tokens_seen": 1998848000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001440325326271988,
+      "loss": 2.6012,
+      "theoretical_loss": 3.4300868770127213,
+      "tokens_seen": 2002124800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014344146018536034,
+      "loss": 2.5659,
+      "theoretical_loss": 3.4296193901368577,
+      "tokens_seen": 2005401600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014285038774352186,
+      "loss": 2.5977,
+      "theoretical_loss": 3.429152879990263,
+      "tokens_seen": 2008678400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014225931530168338,
+      "loss": 2.566,
+      "theoretical_loss": 3.428687342943866,
+      "tokens_seen": 2011955200
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001416682428598449,
+      "loss": 2.5279,
+      "theoretical_loss": 3.428222775387959,
+      "tokens_seen": 2015232000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014107717041800645,
+      "loss": 2.5742,
+      "theoretical_loss": 3.4277591737320696,
+      "tokens_seen": 2018508800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014048609797616795,
+      "loss": 2.5485,
+      "theoretical_loss": 3.4272965344048214,
+      "tokens_seen": 2021785600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013989502553432947,
+      "loss": 2.5261,
+      "theoretical_loss": 3.426834853853804,
+      "tokens_seen": 2025062400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013930395309249102,
+      "loss": 2.5295,
+      "theoretical_loss": 3.4263741285454428,
+      "tokens_seen": 2028339200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013871288065065254,
+      "loss": 2.5315,
+      "theoretical_loss": 3.4259143549648674,
+      "tokens_seen": 2031616000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013812180820881406,
+      "loss": 2.5475,
+      "theoretical_loss": 3.4254555296157827,
+      "tokens_seen": 2034892800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001375307357669756,
+      "loss": 2.5511,
+      "theoretical_loss": 3.424997649020341,
+      "tokens_seen": 2038169600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013693966332513713,
+      "loss": 2.5777,
+      "theoretical_loss": 3.4245407097190173,
+      "tokens_seen": 2041446400
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013634859088329865,
+      "loss": 2.5882,
+      "theoretical_loss": 3.4240847082704793,
+      "tokens_seen": 2044723200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001357575184414602,
+      "loss": 2.5742,
+      "theoretical_loss": 3.423629641251467,
+      "tokens_seen": 2048000000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013516644599962171,
+      "loss": 2.6254,
+      "theoretical_loss": 3.4231755052566673,
+      "tokens_seen": 2051276800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013457537355778324,
+      "loss": 2.6144,
+      "theoretical_loss": 3.4227222968985904,
+      "tokens_seen": 2054553600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013398430111594478,
+      "loss": 2.626,
+      "theoretical_loss": 3.4222700128074495,
+      "tokens_seen": 2057830400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001333932286741063,
+      "loss": 2.5724,
+      "theoretical_loss": 3.4218186496310414,
+      "tokens_seen": 2061107200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013280215623226783,
+      "loss": 2.5903,
+      "theoretical_loss": 3.421368204034624,
+      "tokens_seen": 2064384000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013221108379042937,
+      "loss": 2.6009,
+      "theoretical_loss": 3.4209186727008,
+      "tokens_seen": 2067660800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001316200113485909,
+      "loss": 2.6002,
+      "theoretical_loss": 3.4204700523293985,
+      "tokens_seen": 2070937600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013102893890675241,
+      "loss": 2.58,
+      "theoretical_loss": 3.42002233963736,
+      "tokens_seen": 2074214400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013043786646491394,
+      "loss": 2.5844,
+      "theoretical_loss": 3.4195755313586176,
+      "tokens_seen": 2077491200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012984679402307548,
+      "loss": 2.5175,
+      "theoretical_loss": 3.419129624243986,
+      "tokens_seen": 2080768000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.000129255721581237,
+      "loss": 2.512,
+      "theoretical_loss": 3.4186846150610464,
+      "tokens_seen": 2084044800
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012866464913939852,
+      "loss": 2.5156,
+      "theoretical_loss": 3.418240500594031,
+      "tokens_seen": 2087321600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012807357669756007,
+      "loss": 2.5073,
+      "theoretical_loss": 3.417797277643717,
+      "tokens_seen": 2090598400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001274825042557216,
+      "loss": 2.5129,
+      "theoretical_loss": 3.417354943027311,
+      "tokens_seen": 2093875200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012689143181388311,
+      "loss": 2.4917,
+      "theoretical_loss": 3.41691349357834,
+      "tokens_seen": 2097152000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012630035937204464,
+      "loss": 2.5293,
+      "theoretical_loss": 3.416472926146545,
+      "tokens_seen": 2100428800
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012570928693020616,
+      "loss": 2.5308,
+      "theoretical_loss": 3.4160332375977687,
+      "tokens_seen": 2103705600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012511821448836768,
+      "loss": 2.5132,
+      "theoretical_loss": 3.4155944248138512,
+      "tokens_seen": 2106982400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012452714204652922,
+      "loss": 2.5302,
+      "theoretical_loss": 3.415156484692524,
+      "tokens_seen": 2110259200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012393606960469075,
+      "loss": 2.5203,
+      "theoretical_loss": 3.4147194141473003,
+      "tokens_seen": 2113536000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001233449971628523,
+      "loss": 2.5005,
+      "theoretical_loss": 3.414283210107377,
+      "tokens_seen": 2116812800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012275392472101381,
+      "loss": 2.5163,
+      "theoretical_loss": 3.4138478695175243,
+      "tokens_seen": 2120089600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012216285227917533,
+      "loss": 2.53,
+      "theoretical_loss": 3.413413389337988,
+      "tokens_seen": 2123366400
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012157177983733686,
+      "loss": 2.5193,
+      "theoretical_loss": 3.4129797665443826,
+      "tokens_seen": 2126643200
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/docs_used": 1174903,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5076944828033447,
+      "objective/train/theoretical_loss": 3.4125469981275955,
+      "objective/train/tokens_used": 2150380000,
+      "theoretical_loss": 3.4125469981275955,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012098070739549839,
+      "loss": 2.5596,
+      "theoretical_loss": 3.4125469981275955,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012038963495365992,
+      "loss": 2.52,
+      "theoretical_loss": 3.4121150810936816,
+      "tokens_seen": 2133196800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011981038396065821,
+      "loss": 2.5458,
+      "theoretical_loss": 3.411684012463766,
+      "tokens_seen": 2136473600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011921931151881974,
+      "loss": 2.556,
+      "theoretical_loss": 3.411253789273946,
+      "tokens_seen": 2139750400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011862823907698128,
+      "loss": 2.5514,
+      "theoretical_loss": 3.4108244085751904,
+      "tokens_seen": 2143027200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001180371666351428,
+      "loss": 2.5458,
+      "theoretical_loss": 3.410395867433245,
+      "tokens_seen": 2146304000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011744609419330433,
+      "loss": 2.5532,
+      "theoretical_loss": 3.409968162928534,
+      "tokens_seen": 2149580800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011685502175146587,
+      "loss": 2.5741,
+      "theoretical_loss": 3.4095412921560655,
+      "tokens_seen": 2152857600
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011626394930962739,
+      "loss": 2.5723,
+      "theoretical_loss": 3.4091152522253374,
+      "tokens_seen": 2156134400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011567287686778892,
+      "loss": 2.539,
+      "theoretical_loss": 3.40869004026024,
+      "tokens_seen": 2159411200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011508180442595046,
+      "loss": 2.5796,
+      "theoretical_loss": 3.408265653398967,
+      "tokens_seen": 2162688000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011449073198411198,
+      "loss": 2.5445,
+      "theoretical_loss": 3.40784208879392,
+      "tokens_seen": 2165964800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011389965954227351,
+      "loss": 2.5977,
+      "theoretical_loss": 3.407419343611616,
+      "tokens_seen": 2169241600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011330858710043502,
+      "loss": 2.5833,
+      "theoretical_loss": 3.406997415032599,
+      "tokens_seen": 2172518400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011271751465859655,
+      "loss": 2.5666,
+      "theoretical_loss": 3.406576300251348,
+      "tokens_seen": 2175795200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011212644221675809,
+      "loss": 2.5548,
+      "theoretical_loss": 3.4061559964761856,
+      "tokens_seen": 2179072000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011153536977491961,
+      "loss": 2.5559,
+      "theoretical_loss": 3.4057365009291924,
+      "tokens_seen": 2182348800
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011094429733308114,
+      "loss": 2.4952,
+      "theoretical_loss": 3.405317810846116,
+      "tokens_seen": 2185625600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011035322489124268,
+      "loss": 2.5181,
+      "theoretical_loss": 3.404899923476285,
+      "tokens_seen": 2188902400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001097621524494042,
+      "loss": 2.5382,
+      "theoretical_loss": 3.404482836082521,
+      "tokens_seen": 2192179200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010917108000756573,
+      "loss": 2.5444,
+      "theoretical_loss": 3.4040665459410517,
+      "tokens_seen": 2195456000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010858000756572725,
+      "loss": 2.5389,
+      "theoretical_loss": 3.4036510503414283,
+      "tokens_seen": 2198732800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010798893512388879,
+      "loss": 2.5676,
+      "theoretical_loss": 3.4032363465864375,
+      "tokens_seen": 2202009600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010739786268205032,
+      "loss": 2.5517,
+      "theoretical_loss": 3.402822431992018,
+      "tokens_seen": 2205286400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010680679024021184,
+      "loss": 2.5663,
+      "theoretical_loss": 3.402409303887179,
+      "tokens_seen": 2208563200
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010621571779837336,
+      "loss": 2.5307,
+      "theoretical_loss": 3.401996959613914,
+      "tokens_seen": 2211840000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001056246453565349,
+      "loss": 2.4961,
+      "theoretical_loss": 3.4015853965271217,
+      "tokens_seen": 2215116800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010503357291469642,
+      "loss": 2.5326,
+      "theoretical_loss": 3.4011746119945214,
+      "tokens_seen": 2218393600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010444250047285795,
+      "loss": 2.5381,
+      "theoretical_loss": 3.4007646033965746,
+      "tokens_seen": 2221670400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010385142803101949,
+      "loss": 2.5942,
+      "theoretical_loss": 3.4003553681264025,
+      "tokens_seen": 2224947200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010326035558918101,
+      "loss": 2.5615,
+      "theoretical_loss": 3.399946903589709,
+      "tokens_seen": 2228224000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010266928314734254,
+      "loss": 2.5462,
+      "theoretical_loss": 3.3995392072046986,
+      "tokens_seen": 2231500800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010207821070550406,
+      "loss": 2.54,
+      "theoretical_loss": 3.3991322764019993,
+      "tokens_seen": 2234777600
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001014871382636656,
+      "loss": 2.5474,
+      "theoretical_loss": 3.398726108624585,
+      "tokens_seen": 2238054400
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010089606582182713,
+      "loss": 2.5188,
+      "theoretical_loss": 3.3983207013276977,
+      "tokens_seen": 2241331200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010030499337998865,
+      "loss": 2.5264,
+      "theoretical_loss": 3.3979160519787706,
+      "tokens_seen": 2244608000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.971392093815019e-05,
+      "loss": 2.5565,
+      "theoretical_loss": 3.397512158057353,
+      "tokens_seen": 2247884800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.912284849631172e-05,
+      "loss": 2.5332,
+      "theoretical_loss": 3.3971090170550333,
+      "tokens_seen": 2251161600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.853177605447323e-05,
+      "loss": 2.5643,
+      "theoretical_loss": 3.396706626475366,
+      "tokens_seen": 2254438400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.794070361263476e-05,
+      "loss": 2.5466,
+      "theoretical_loss": 3.396304983833794,
+      "tokens_seen": 2257715200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.73496311707963e-05,
+      "loss": 2.5491,
+      "theoretical_loss": 3.3959040866575805,
+      "tokens_seen": 2260992000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.675855872895782e-05,
+      "loss": 2.6207,
+      "theoretical_loss": 3.3955039324857292,
+      "tokens_seen": 2264268800
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.616748628711935e-05,
+      "loss": 2.7366,
+      "theoretical_loss": 3.395104518868916,
+      "tokens_seen": 2267545600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.557641384528087e-05,
+      "loss": 2.7848,
+      "theoretical_loss": 3.3947058433694157,
+      "tokens_seen": 2270822400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.498534140344241e-05,
+      "loss": 2.8015,
+      "theoretical_loss": 3.39430790356103,
+      "tokens_seen": 2274099200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.439426896160394e-05,
+      "loss": 2.7401,
+      "theoretical_loss": 3.3939106970290163,
+      "tokens_seen": 2277376000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.380319651976546e-05,
+      "loss": 2.7222,
+      "theoretical_loss": 3.3935142213700185,
+      "tokens_seen": 2280652800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.3212124077927e-05,
+      "loss": 2.7245,
+      "theoretical_loss": 3.393118474191996,
+      "tokens_seen": 2283929600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.262105163608853e-05,
+      "loss": 2.6815,
+      "theoretical_loss": 3.392723453114155,
+      "tokens_seen": 2287206400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.202997919425005e-05,
+      "loss": 2.6319,
+      "theoretical_loss": 3.3923291557668778,
+      "tokens_seen": 2290483200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4293460113722673,
+      "debugging/distinct-1-grams": 0.7974843612112625,
+      "debugging/distinct-2-grams": 0.9559654625250797,
+      "debugging/entropy-1-grams": 5.759683494668553,
+      "debugging/entropy-2-grams": 6.4603299955048765,
+      "debugging/length": 457.6363636363636,
+      "debugging/num_segments": 11,
+      "epoch": 0.82,
+      "objective/train/docs_used": 1259077,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5392394065856934,
+      "objective/train/theoretical_loss": 3.391935579791658,
+      "objective/train/tokens_used": 2314220000,
+      "theoretical_loss": 3.391935579791658,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.143890675241157e-05,
+      "loss": 2.6025,
+      "theoretical_loss": 3.391935579791658,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.08478343105731e-05,
+      "loss": 2.6059,
+      "theoretical_loss": 3.3915427228410278,
+      "tokens_seen": 2297036800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.025676186873463e-05,
+      "loss": 2.5726,
+      "theoretical_loss": 3.391150582578495,
+      "tokens_seen": 2300313600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.966568942689616e-05,
+      "loss": 2.5623,
+      "theoretical_loss": 3.390759156678474,
+      "tokens_seen": 2303590400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.907461698505768e-05,
+      "loss": 2.5695,
+      "theoretical_loss": 3.3903684428262197,
+      "tokens_seen": 2306867200
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.848354454321922e-05,
+      "loss": 2.5468,
+      "theoretical_loss": 3.3899784387177623,
+      "tokens_seen": 2310144000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.789247210138075e-05,
+      "loss": 2.5231,
+      "theoretical_loss": 3.3895891420598403,
+      "tokens_seen": 2313420800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.730139965954227e-05,
+      "loss": 2.5267,
+      "theoretical_loss": 3.3892005505698384,
+      "tokens_seen": 2316697600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.671032721770381e-05,
+      "loss": 2.5361,
+      "theoretical_loss": 3.3888126619757197,
+      "tokens_seen": 2319974400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.611925477586534e-05,
+      "loss": 2.5273,
+      "theoretical_loss": 3.388425474015966,
+      "tokens_seen": 2323251200
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.552818233402686e-05,
+      "loss": 2.5433,
+      "theoretical_loss": 3.3880389844395093,
+      "tokens_seen": 2326528000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.49371098921884e-05,
+      "loss": 2.4981,
+      "theoretical_loss": 3.3876531910056733,
+      "tokens_seen": 2329804800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.43460374503499e-05,
+      "loss": 2.5395,
+      "theoretical_loss": 3.387268091484109,
+      "tokens_seen": 2333081600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.375496500851144e-05,
+      "loss": 2.5397,
+      "theoretical_loss": 3.386883683654732,
+      "tokens_seen": 2336358400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.316389256667297e-05,
+      "loss": 2.535,
+      "theoretical_loss": 3.386499965307662,
+      "tokens_seen": 2339635200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.25728201248345e-05,
+      "loss": 2.5006,
+      "theoretical_loss": 3.3861169342431627,
+      "tokens_seen": 2342912000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.198174768299603e-05,
+      "loss": 2.5626,
+      "theoretical_loss": 3.3857345882715784,
+      "tokens_seen": 2346188800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.139067524115756e-05,
+      "loss": 2.5485,
+      "theoretical_loss": 3.3853529252132755,
+      "tokens_seen": 2349465600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.079960279931908e-05,
+      "loss": 2.5578,
+      "theoretical_loss": 3.384971942898583,
+      "tokens_seen": 2352742400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.020853035748062e-05,
+      "loss": 2.5631,
+      "theoretical_loss": 3.384591639167732,
+      "tokens_seen": 2356019200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.961745791564215e-05,
+      "loss": 2.5817,
+      "theoretical_loss": 3.3842120118707983,
+      "tokens_seen": 2359296000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.902638547380367e-05,
+      "loss": 2.6779,
+      "theoretical_loss": 3.383833058867643,
+      "tokens_seen": 2362572800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.843531303196521e-05,
+      "loss": 2.7095,
+      "theoretical_loss": 3.3834547780278537,
+      "tokens_seen": 2365849600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.784424059012673e-05,
+      "loss": 2.6822,
+      "theoretical_loss": 3.3830771672306885,
+      "tokens_seen": 2369126400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.725316814828825e-05,
+      "loss": 2.6745,
+      "theoretical_loss": 3.382700224365018,
+      "tokens_seen": 2372403200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.666209570644978e-05,
+      "loss": 2.6195,
+      "theoretical_loss": 3.382323947329268,
+      "tokens_seen": 2375680000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.60710232646113e-05,
+      "loss": 2.6191,
+      "theoretical_loss": 3.3819483340313647,
+      "tokens_seen": 2378956800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.547995082277284e-05,
+      "loss": 2.5827,
+      "theoretical_loss": 3.3815733823886776,
+      "tokens_seen": 2382233600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.488887838093437e-05,
+      "loss": 2.5749,
+      "theoretical_loss": 3.381199090327963,
+      "tokens_seen": 2385510400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.42978059390959e-05,
+      "loss": 2.5639,
+      "theoretical_loss": 3.3808254557853115,
+      "tokens_seen": 2388787200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.370673349725743e-05,
+      "loss": 2.5416,
+      "theoretical_loss": 3.3804524767060906,
+      "tokens_seen": 2392064000
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.311566105541895e-05,
+      "loss": 2.5528,
+      "theoretical_loss": 3.3800801510448917,
+      "tokens_seen": 2395340800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.252458861358048e-05,
+      "loss": 2.5388,
+      "theoretical_loss": 3.379708476765477,
+      "tokens_seen": 2398617600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.193351617174202e-05,
+      "loss": 2.5428,
+      "theoretical_loss": 3.3793374518407235,
+      "tokens_seen": 2401894400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.134244372990354e-05,
+      "loss": 2.5699,
+      "theoretical_loss": 3.3789670742525715,
+      "tokens_seen": 2405171200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.075137128806507e-05,
+      "loss": 2.5799,
+      "theoretical_loss": 3.3785973419919717,
+      "tokens_seen": 2408448000
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.01602988462266e-05,
+      "loss": 2.5551,
+      "theoretical_loss": 3.3782282530588335,
+      "tokens_seen": 2411724800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.956922640438811e-05,
+      "loss": 2.5443,
+      "theoretical_loss": 3.3778598054619695,
+      "tokens_seen": 2415001600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.897815396254965e-05,
+      "loss": 2.5418,
+      "theoretical_loss": 3.377491997219049,
+      "tokens_seen": 2418278400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.838708152071118e-05,
+      "loss": 2.6263,
+      "theoretical_loss": 3.3771248263565425,
+      "tokens_seen": 2421555200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.77960090788727e-05,
+      "loss": 2.6284,
+      "theoretical_loss": 3.376758290909673,
+      "tokens_seen": 2424832000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.720493663703424e-05,
+      "loss": 2.5957,
+      "theoretical_loss": 3.3763923889223646,
+      "tokens_seen": 2428108800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.661386419519576e-05,
+      "loss": 2.6041,
+      "theoretical_loss": 3.3760271184471917,
+      "tokens_seen": 2431385600
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.602279175335729e-05,
+      "loss": 2.5739,
+      "theoretical_loss": 3.3756624775453306,
+      "tokens_seen": 2434662400
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.543171931151883e-05,
+      "loss": 2.5674,
+      "theoretical_loss": 3.3752984642865087,
+      "tokens_seen": 2437939200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.484064686968035e-05,
+      "loss": 2.5583,
+      "theoretical_loss": 3.374935076748957,
+      "tokens_seen": 2441216000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.424957442784188e-05,
+      "loss": 2.5728,
+      "theoretical_loss": 3.374572313019359,
+      "tokens_seen": 2444492800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.365850198600342e-05,
+      "loss": 2.5836,
+      "theoretical_loss": 3.3742101711928036,
+      "tokens_seen": 2447769600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.306742954416494e-05,
+      "loss": 2.6035,
+      "theoretical_loss": 3.3738486493727375,
+      "tokens_seen": 2451046400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.247635710232646e-05,
+      "loss": 2.5751,
+      "theoretical_loss": 3.3734877456709165,
+      "tokens_seen": 2454323200
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/docs_used": 1345695,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5207114219665527,
+      "objective/train/theoretical_loss": 3.373127458207357,
+      "objective/train/tokens_used": 2478060000,
+      "theoretical_loss": 3.373127458207357,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.188528466048799e-05,
+      "loss": 2.5957,
+      "theoretical_loss": 3.373127458207357,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.129421221864951e-05,
+      "loss": 2.5463,
+      "theoretical_loss": 3.372767785110293,
+      "tokens_seen": 2460876800
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.070313977681105e-05,
+      "loss": 2.5747,
+      "theoretical_loss": 3.3724087245161227,
+      "tokens_seen": 2464153600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0112067334972575e-05,
+      "loss": 2.5785,
+      "theoretical_loss": 3.3720502745693697,
+      "tokens_seen": 2467430400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.95209948931341e-05,
+      "loss": 2.5468,
+      "theoretical_loss": 3.37169243342263,
+      "tokens_seen": 2470707200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.892992245129563e-05,
+      "loss": 2.5153,
+      "theoretical_loss": 3.3713351992365315,
+      "tokens_seen": 2473984000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8338850009457165e-05,
+      "loss": 2.4929,
+      "theoretical_loss": 3.3709785701796844,
+      "tokens_seen": 2477260800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.774777756761869e-05,
+      "loss": 2.5351,
+      "theoretical_loss": 3.3706225444286395,
+      "tokens_seen": 2480537600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.715670512578021e-05,
+      "loss": 2.5328,
+      "theoretical_loss": 3.3702671201678407,
+      "tokens_seen": 2483814400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.656563268394174e-05,
+      "loss": 2.5478,
+      "theoretical_loss": 3.3699122955895815,
+      "tokens_seen": 2487091200
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5974560242103275e-05,
+      "loss": 2.5223,
+      "theoretical_loss": 3.3695580688939617,
+      "tokens_seen": 2490368000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.53834878002648e-05,
+      "loss": 2.5421,
+      "theoretical_loss": 3.3692044382888415,
+      "tokens_seen": 2493644800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.479241535842633e-05,
+      "loss": 2.5288,
+      "theoretical_loss": 3.368851401989799,
+      "tokens_seen": 2496921600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.420134291658786e-05,
+      "loss": 2.5443,
+      "theoretical_loss": 3.3684989582200884,
+      "tokens_seen": 2500198400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.3610270474749385e-05,
+      "loss": 2.5078,
+      "theoretical_loss": 3.368147105210593,
+      "tokens_seen": 2503475200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.301919803291091e-05,
+      "loss": 2.5376,
+      "theoretical_loss": 3.3677958411997873,
+      "tokens_seen": 2506752000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.242812559107244e-05,
+      "loss": 2.5017,
+      "theoretical_loss": 3.36744516443369,
+      "tokens_seen": 2510028800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1837053149233975e-05,
+      "loss": 2.4733,
+      "theoretical_loss": 3.3670950731658253,
+      "tokens_seen": 2513305600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.12459807073955e-05,
+      "loss": 2.5319,
+      "theoretical_loss": 3.36674556565718,
+      "tokens_seen": 2516582400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.065490826555703e-05,
+      "loss": 2.5157,
+      "theoretical_loss": 3.3663966401761614,
+      "tokens_seen": 2519859200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.006383582371855e-05,
+      "loss": 2.5249,
+      "theoretical_loss": 3.3660482949985555,
+      "tokens_seen": 2523136000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9472763381880085e-05,
+      "loss": 2.5276,
+      "theoretical_loss": 3.3657005284074883,
+      "tokens_seen": 2526412800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.888169094004161e-05,
+      "loss": 2.5394,
+      "theoretical_loss": 3.3653533386933834,
+      "tokens_seen": 2529689600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.829061849820314e-05,
+      "loss": 2.5217,
+      "theoretical_loss": 3.3650067241539205,
+      "tokens_seen": 2532966400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.769954605636467e-05,
+      "loss": 2.5269,
+      "theoretical_loss": 3.364660683093999,
+      "tokens_seen": 2536243200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.71084736145262e-05,
+      "loss": 2.4955,
+      "theoretical_loss": 3.364315213825694,
+      "tokens_seen": 2539520000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.651740117268772e-05,
+      "loss": 2.5233,
+      "theoretical_loss": 3.3639703146682187,
+      "tokens_seen": 2542796800
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.592632873084925e-05,
+      "loss": 2.4791,
+      "theoretical_loss": 3.3636259839478853,
+      "tokens_seen": 2546073600
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.533525628901078e-05,
+      "loss": 2.4998,
+      "theoretical_loss": 3.363282219998065,
+      "tokens_seen": 2549350400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.474418384717231e-05,
+      "loss": 2.5055,
+      "theoretical_loss": 3.3629390211591494,
+      "tokens_seen": 2552627200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.415311140533384e-05,
+      "loss": 2.4614,
+      "theoretical_loss": 3.3625963857785135,
+      "tokens_seen": 2555904000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.356203896349537e-05,
+      "loss": 2.4681,
+      "theoretical_loss": 3.3622543122104744,
+      "tokens_seen": 2559180800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2970966521656895e-05,
+      "loss": 2.5063,
+      "theoretical_loss": 3.361912798816256,
+      "tokens_seen": 2562457600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.237989407981842e-05,
+      "loss": 2.4647,
+      "theoretical_loss": 3.36157184396395,
+      "tokens_seen": 2565734400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.178882163797995e-05,
+      "loss": 2.4559,
+      "theoretical_loss": 3.3612314460284782,
+      "tokens_seen": 2569011200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.119774919614148e-05,
+      "loss": 2.5082,
+      "theoretical_loss": 3.360891603391557,
+      "tokens_seen": 2572288000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.060667675430301e-05,
+      "loss": 2.5179,
+      "theoretical_loss": 3.3605523144416574,
+      "tokens_seen": 2575564800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.001560431246454e-05,
+      "loss": 2.4835,
+      "theoretical_loss": 3.3602135775739708,
+      "tokens_seen": 2578841600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.942453187062606e-05,
+      "loss": 2.5337,
+      "theoretical_loss": 3.3598753911903705,
+      "tokens_seen": 2582118400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.883345942878759e-05,
+      "loss": 2.548,
+      "theoretical_loss": 3.359537753699379,
+      "tokens_seen": 2585395200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8254208435785894e-05,
+      "loss": 2.5126,
+      "theoretical_loss": 3.3592006635161265,
+      "tokens_seen": 2588672000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.766313599394742e-05,
+      "loss": 2.3978,
+      "theoretical_loss": 3.35886411906232,
+      "tokens_seen": 2591948800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.707206355210894e-05,
+      "loss": 2.432,
+      "theoretical_loss": 3.358528118766205,
+      "tokens_seen": 2595225600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6480991110270477e-05,
+      "loss": 2.4266,
+      "theoretical_loss": 3.3581926610625317,
+      "tokens_seen": 2598502400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5889918668432004e-05,
+      "loss": 2.4904,
+      "theoretical_loss": 3.357857744392518,
+      "tokens_seen": 2601779200
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.529884622659353e-05,
+      "loss": 2.4821,
+      "theoretical_loss": 3.3575233672038176,
+      "tokens_seen": 2605056000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.470777378475506e-05,
+      "loss": 2.4328,
+      "theoretical_loss": 3.3571895279504824,
+      "tokens_seen": 2608332800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4116701342916594e-05,
+      "loss": 2.4331,
+      "theoretical_loss": 3.3568562250929306,
+      "tokens_seen": 2611609600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3525628901078114e-05,
+      "loss": 2.454,
+      "theoretical_loss": 3.3565234570979094,
+      "tokens_seen": 2614886400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.293455645923964e-05,
+      "loss": 2.4991,
+      "theoretical_loss": 3.356191222438466,
+      "tokens_seen": 2618163200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4959272434748912,
+      "debugging/distinct-1-grams": 0.77233460946343,
+      "debugging/distinct-2-grams": 0.965889219747869,
+      "debugging/entropy-1-grams": 5.680919515599907,
+      "debugging/entropy-2-grams": 6.451736434336183,
+      "debugging/length": 541.7777777777778,
+      "debugging/num_segments": 9,
+      "epoch": 0.94,
+      "objective/train/docs_used": 1436745,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4117465019226074,
+      "objective/train/theoretical_loss": 3.355859519593907,
+      "objective/train/tokens_used": 2641900000,
+      "theoretical_loss": 3.355859519593907,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.234348401740117e-05,
+      "loss": 2.4485,
+      "theoretical_loss": 3.355859519593907,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1752411575562704e-05,
+      "loss": 2.4426,
+      "theoretical_loss": 3.355528347049773,
+      "tokens_seen": 2624716800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.116133913372423e-05,
+      "loss": 2.5038,
+      "theoretical_loss": 3.355197703297798,
+      "tokens_seen": 2627993600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.057026669188576e-05,
+      "loss": 2.4836,
+      "theoretical_loss": 3.354867586835882,
+      "tokens_seen": 2631270400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9979194250047287e-05,
+      "loss": 2.4733,
+      "theoretical_loss": 3.354537996168054,
+      "tokens_seen": 2634547200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9388121808208818e-05,
+      "loss": 2.4474,
+      "theoretical_loss": 3.3542089298044413,
+      "tokens_seen": 2637824000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8797049366370342e-05,
+      "loss": 2.4374,
+      "theoretical_loss": 3.3538803862612374,
+      "tokens_seen": 2641100800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8205976924531873e-05,
+      "loss": 2.4557,
+      "theoretical_loss": 3.3535523640606697,
+      "tokens_seen": 2644377600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.76149044826934e-05,
+      "loss": 2.5111,
+      "theoretical_loss": 3.353224861730966,
+      "tokens_seen": 2647654400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7023832040854928e-05,
+      "loss": 2.4421,
+      "theoretical_loss": 3.3528978778063254,
+      "tokens_seen": 2650931200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6432759599016455e-05,
+      "loss": 2.4618,
+      "theoretical_loss": 3.3525714108268834,
+      "tokens_seen": 2654208000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5841687157177986e-05,
+      "loss": 2.486,
+      "theoretical_loss": 3.352245459338684,
+      "tokens_seen": 2657484800
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.525061471533951e-05,
+      "loss": 2.4714,
+      "theoretical_loss": 3.351920021893646,
+      "tokens_seen": 2660761600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.465954227350104e-05,
+      "loss": 2.5016,
+      "theoretical_loss": 3.3515950970495334,
+      "tokens_seen": 2664038400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.406846983166257e-05,
+      "loss": 2.4939,
+      "theoretical_loss": 3.3512706833699237,
+      "tokens_seen": 2667315200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3477397389824097e-05,
+      "loss": 2.4871,
+      "theoretical_loss": 3.3509467794241794,
+      "tokens_seen": 2670592000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2886324947985624e-05,
+      "loss": 2.5656,
+      "theoretical_loss": 3.350623383787414,
+      "tokens_seen": 2673868800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2295252506147155e-05,
+      "loss": 2.5328,
+      "theoretical_loss": 3.350300495040466,
+      "tokens_seen": 2677145600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1704180064308683e-05,
+      "loss": 2.5484,
+      "theoretical_loss": 3.3499781117698646,
+      "tokens_seen": 2680422400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.111310762247021e-05,
+      "loss": 2.5503,
+      "theoretical_loss": 3.349656232567805,
+      "tokens_seen": 2683699200
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.052203518063174e-05,
+      "loss": 2.5273,
+      "theoretical_loss": 3.3493348560321143,
+      "tokens_seen": 2686976000
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9930962738793265e-05,
+      "loss": 2.5443,
+      "theoretical_loss": 3.349013980766224,
+      "tokens_seen": 2690252800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9339890296954796e-05,
+      "loss": 2.5656,
+      "theoretical_loss": 3.3486936053791423,
+      "tokens_seen": 2693529600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8748817855116324e-05,
+      "loss": 2.5167,
+      "theoretical_loss": 3.348373728485421,
+      "tokens_seen": 2696806400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.815774541327785e-05,
+      "loss": 2.5093,
+      "theoretical_loss": 3.348054348705131,
+      "tokens_seen": 2700083200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.756667297143938e-05,
+      "loss": 2.5239,
+      "theoretical_loss": 3.3477354646638315,
+      "tokens_seen": 2703360000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.697560052960091e-05,
+      "loss": 2.5361,
+      "theoretical_loss": 3.3474170749925416,
+      "tokens_seen": 2706636800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6384528087762434e-05,
+      "loss": 2.5378,
+      "theoretical_loss": 3.347099178327713,
+      "tokens_seen": 2709913600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5793455645923965e-05,
+      "loss": 2.5255,
+      "theoretical_loss": 3.3467817733112,
+      "tokens_seen": 2713190400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5202383204085493e-05,
+      "loss": 2.5046,
+      "theoretical_loss": 3.3464648585902346,
+      "tokens_seen": 2716467200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4611310762247022e-05,
+      "loss": 2.5056,
+      "theoretical_loss": 3.3461484328173956,
+      "tokens_seen": 2719744000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.402023832040855e-05,
+      "loss": 2.4826,
+      "theoretical_loss": 3.3458324946505824,
+      "tokens_seen": 2723020800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3440987327406847e-05,
+      "loss": 2.5159,
+      "theoretical_loss": 3.3455170427529883,
+      "tokens_seen": 2726297600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2849914885568376e-05,
+      "loss": 2.49,
+      "theoretical_loss": 3.345202075793073,
+      "tokens_seen": 2729574400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2258842443729904e-05,
+      "loss": 2.5143,
+      "theoretical_loss": 3.344887592444535,
+      "tokens_seen": 2732851200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1667770001891433e-05,
+      "loss": 2.5255,
+      "theoretical_loss": 3.3445735913862835,
+      "tokens_seen": 2736128000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.107669756005296e-05,
+      "loss": 2.5153,
+      "theoretical_loss": 3.344260071302415,
+      "tokens_seen": 2739404800
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0485625118214488e-05,
+      "loss": 2.529,
+      "theoretical_loss": 3.3439470308821844,
+      "tokens_seen": 2742681600
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.894552676376017e-06,
+      "loss": 2.5377,
+      "theoretical_loss": 3.3436344688199786,
+      "tokens_seen": 2745958400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.303480234537545e-06,
+      "loss": 2.5329,
+      "theoretical_loss": 3.3433223838152917,
+      "tokens_seen": 2749235200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.712407792699074e-06,
+      "loss": 2.506,
+      "theoretical_loss": 3.3430107745726976,
+      "tokens_seen": 2752512000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.121335350860602e-06,
+      "loss": 2.4914,
+      "theoretical_loss": 3.3426996398018245,
+      "tokens_seen": 2755788800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.530262909022129e-06,
+      "loss": 2.5027,
+      "theoretical_loss": 3.3423889782173304,
+      "tokens_seen": 2759065600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.939190467183658e-06,
+      "loss": 2.4868,
+      "theoretical_loss": 3.342078788538875,
+      "tokens_seen": 2762342400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.348118025345187e-06,
+      "loss": 2.4896,
+      "theoretical_loss": 3.3417690694910975,
+      "tokens_seen": 2765619200
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.757045583506715e-06,
+      "loss": 2.4767,
+      "theoretical_loss": 3.3414598198035885,
+      "tokens_seen": 2768896000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.165973141668243e-06,
+      "loss": 2.4887,
+      "theoretical_loss": 3.3411510382108665,
+      "tokens_seen": 2772172800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.574900699829771e-06,
+      "loss": 2.4898,
+      "theoretical_loss": 3.340842723452353,
+      "tokens_seen": 2775449600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.9838282579913e-06,
+      "loss": 2.4962,
+      "theoretical_loss": 3.340534874272347,
+      "tokens_seen": 2778726400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.392755816152828e-06,
+      "loss": 2.5,
+      "theoretical_loss": 3.340227489420001,
+      "tokens_seen": 2782003200
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/docs_used": 1527604,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.3713839054107666,
+      "objective/train/theoretical_loss": 3.339920567649296,
+      "objective/train/tokens_used": 2805740000,
+      "theoretical_loss": 3.339920567649296,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.801683374314356e-06,
+      "loss": 2.4823,
+      "theoretical_loss": 3.339920567649296,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.210610932475884e-06,
+      "loss": 2.4942,
+      "theoretical_loss": 3.339614107719018,
+      "tokens_seen": 2788556800
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6195384906374124e-06,
+      "loss": 2.544,
+      "theoretical_loss": 3.339308108392733,
+      "tokens_seen": 2791833600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0284660487989408e-06,
+      "loss": 2.5998,
+      "theoretical_loss": 3.3390025684387643,
+      "tokens_seen": 2795110400
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.373936069604691e-07,
+      "loss": 2.5809,
+      "theoretical_loss": 3.3386974866301666,
+      "tokens_seen": 2798387200
+    }
+  ],
+  "max_steps": 42724,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.428920419221504e+18,
+  "trial_name": null,
+  "trial_params": null
+}