diff --git "a/checkpoint-42720/trainer_state.json" "b/checkpoint-42720/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-42720/trainer_state.json"
@@ -0,0 +1,6312 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9999063758075086,
+  "global_step": 42720,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1682242990654204e-06,
+      "loss": 4.3456,
+      "theoretical_loss": 20.81281780154715,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.841121495327103e-05,
+      "loss": 3.7625,
+      "theoretical_loss": 8.563482664611069,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011682242990654206,
+      "loss": 3.0678,
+      "theoretical_loss": 7.4777587180480305,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017523364485981307,
+      "loss": 2.8059,
+      "theoretical_loss": 6.9337544888949,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023364485981308412,
+      "loss": 2.5892,
+      "theoretical_loss": 6.583566228426414,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029205607476635517,
+      "loss": 2.5502,
+      "theoretical_loss": 6.330713565116083,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035046728971962614,
+      "loss": 2.4491,
+      "theoretical_loss": 6.135529231940326,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004088785046728972,
+      "loss": 2.3659,
+      "theoretical_loss": 5.978101583869607,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046728971962616824,
+      "loss": 2.3654,
+      "theoretical_loss": 5.8471173262659235,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499739928125591,
+      "loss": 2.3593,
+      "theoretical_loss": 5.7355768158821245,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991488556837526,
+      "loss": 2.3594,
+      "theoretical_loss": 5.638870144071353,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985577832419141,
+      "loss": 2.3048,
+      "theoretical_loss": 5.553812381844907,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979667108000757,
+      "loss": 2.2536,
+      "theoretical_loss": 5.478118080556438,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973756383582371,
+      "loss": 2.2128,
+      "theoretical_loss": 5.410095959579362,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967845659163987,
+      "loss": 2.2557,
+      "theoretical_loss": 5.348462083735834,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961934934745603,
+      "loss": 2.2013,
+      "theoretical_loss": 5.292220566937567,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956024210327218,
+      "loss": 2.1942,
+      "theoretical_loss": 5.240584625769978,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950113485908833,
+      "loss": 2.1848,
+      "theoretical_loss": 5.192922724525789,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944202761490448,
+      "loss": 2.1625,
+      "theoretical_loss": 5.1487208633564405,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938292037072064,
+      "loss": 2.1605,
+      "theoretical_loss": 5.107555562405102,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932381312653678,
+      "loss": 2.1066,
+      "theoretical_loss": 5.069074117143246,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926470588235294,
+      "loss": 2.1477,
+      "theoretical_loss": 5.032979909838007,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492055986381691,
+      "loss": 2.142,
+      "theoretical_loss": 4.999021308224664,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914649139398525,
+      "loss": 2.1005,
+      "theoretical_loss": 4.966983155351962,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490873841498014,
+      "loss": 2.1206,
+      "theoretical_loss": 4.9366801616251355,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902827690561755,
+      "loss": 2.0318,
+      "theoretical_loss": 4.907951713830082,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896916966143371,
+      "loss": 2.0003,
+      "theoretical_loss": 4.880657753812926,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891006241724985,
+      "loss": 2.0118,
+      "theoretical_loss": 4.854675474481779,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885095517306601,
+      "loss": 2.0002,
+      "theoretical_loss": 4.8298966473088125,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879184792888217,
+      "loss": 2.0062,
+      "theoretical_loss": 4.8062254427779205,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873274068469832,
+      "loss": 1.9869,
+      "theoretical_loss": 4.783576639276257,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048673633440514467,
+      "loss": 2.0435,
+      "theoretical_loss": 4.761874140772408,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861452619633062,
+      "loss": 1.995,
+      "theoretical_loss": 4.741049741962473,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855541895214677,
+      "loss": 1.8767,
+      "theoretical_loss": 4.721042093249051,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849631170796293,
+      "loss": 1.905,
+      "theoretical_loss": 4.701795828231866,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843720446377908,
+      "loss": 1.9278,
+      "theoretical_loss": 4.68326082423593,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048378097219595233,
+      "loss": 1.9084,
+      "theoretical_loss": 4.665391572426282,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048318989975411385,
+      "loss": 1.8945,
+      "theoretical_loss": 4.648146638719739,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048259882731227537,
+      "loss": 1.8564,
+      "theoretical_loss": 4.631488200339643,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820077548704369,
+      "loss": 1.895,
+      "theoretical_loss": 4.615381645715717,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141668242859847,
+      "loss": 1.8264,
+      "theoretical_loss": 4.599795227690505,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048082560998676,
+      "loss": 1.9229,
+      "theoretical_loss": 4.584699761792674,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802345375449215,
+      "loss": 1.8639,
+      "theoretical_loss": 4.570068362778516,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047964346510308303,
+      "loss": 1.8671,
+      "theoretical_loss": 4.555876213804037,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905239266124455,
+      "loss": 1.8481,
+      "theoretical_loss": 4.542100363530799,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047846132021940607,
+      "loss": 1.8247,
+      "theoretical_loss": 4.528719547234816,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778702477775676,
+      "loss": 1.8526,
+      "theoretical_loss": 4.515714028614996,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047727917533572917,
+      "loss": 1.7782,
+      "theoretical_loss": 4.503065459513339,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766881028938907,
+      "loss": 1.8401,
+      "theoretical_loss": 4.4907567551852665,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760970304520522,
+      "loss": 1.7754,
+      "theoretical_loss": 4.478771983111967,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.3757331371307373,
+      "objective/train/avg_token_score": 0.02785523608326912,
+      "objective/train/docs_used": 104000,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.5752174854278564,
+      "objective/train/original_loss": 5.159116268157959,
+      "objective/train/theoretical_loss": 4.467096263641219,
+      "objective/train/tokens_used": 184300000,
+      "objective/train/unlikelihood_loss": 0.05230223760008812,
+      "theoretical_loss": 4.467096263641219,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047550595801021373,
+      "loss": 1.7342,
+      "theoretical_loss": 4.467096263641219,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047491488556837525,
+      "loss": 1.7527,
+      "theoretical_loss": 4.455715680989545,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432381312653677,
+      "loss": 1.7593,
+      "theoretical_loss": 4.44461720334543,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047373274068469835,
+      "loss": 1.7023,
+      "theoretical_loss": 4.433788610987646,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047314166824285987,
+      "loss": 1.6898,
+      "theoretical_loss": 4.42321843148016,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725505958010214,
+      "loss": 1.6724,
+      "theoretical_loss": 4.412895881130142,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719595233591829,
+      "loss": 1.7611,
+      "theoretical_loss": 4.4028108120020795,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047136845091734443,
+      "loss": 1.69,
+      "theoretical_loss": 4.392953663871862,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047077737847550595,
+      "loss": 1.7081,
+      "theoretical_loss": 4.383315420582533,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701863060336675,
+      "loss": 1.723,
+      "theoretical_loss": 4.373887570330275,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046959523359182905,
+      "loss": 1.6931,
+      "theoretical_loss": 4.364662069466704,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046900416114999057,
+      "loss": 1.6707,
+      "theoretical_loss": 4.355631309453283,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684130887081521,
+      "loss": 1.6459,
+      "theoretical_loss": 4.346788086646671,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678220162663136,
+      "loss": 1.6859,
+      "theoretical_loss": 4.33812557463116,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046723094382447513,
+      "loss": 1.6796,
+      "theoretical_loss": 4.329637298846812,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046663987138263665,
+      "loss": 1.7022,
+      "theoretical_loss": 4.321317113290252,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660487989407982,
+      "loss": 1.6707,
+      "theoretical_loss": 4.3131591790897925,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046545772649895975,
+      "loss": 1.6846,
+      "theoretical_loss": 4.305157944778228,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046486665405712127,
+      "loss": 1.7325,
+      "theoretical_loss": 4.297308128105687,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642755816152828,
+      "loss": 1.7026,
+      "theoretical_loss": 4.2896046992515995,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636845091734443,
+      "loss": 1.7491,
+      "theoretical_loss": 4.282042865309616,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046309343673160583,
+      "loss": 1.7436,
+      "theoretical_loss": 4.274618055932298,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625023642897674,
+      "loss": 1.697,
+      "theoretical_loss": 4.267325910033897,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619112918479289,
+      "loss": 1.6645,
+      "theoretical_loss": 4.260162263459744,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046132021940609044,
+      "loss": 1.7309,
+      "theoretical_loss": 4.253123137539814,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046072914696425197,
+      "loss": 1.7716,
+      "theoretical_loss": 4.246204728452055,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601380745224135,
+      "loss": 1.7527,
+      "theoretical_loss": 4.239403397328261,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459547002080575,
+      "loss": 1.7146,
+      "theoretical_loss": 4.232715661041632,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589559296387366,
+      "loss": 1.7026,
+      "theoretical_loss": 4.226138183620867,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583648571968981,
+      "loss": 1.6554,
+      "theoretical_loss": 4.219667768240775,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577737847550596,
+      "loss": 1.6424,
+      "theoretical_loss": 4.213301349743924,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571827123132211,
+      "loss": 1.6548,
+      "theoretical_loss": 4.20703598765197,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565916398713826,
+      "loss": 1.6318,
+      "theoretical_loss": 4.2008688596290025,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045600056742954413,
+      "loss": 1.6147,
+      "theoretical_loss": 4.194797255362549,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554094949877057,
+      "loss": 1.6936,
+      "theoretical_loss": 4.188818570830883,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045481842254586723,
+      "loss": 1.6677,
+      "theoretical_loss": 4.182930302927963,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045422735010402875,
+      "loss": 1.6291,
+      "theoretical_loss": 4.17713004441978,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045363627766219027,
+      "loss": 1.6848,
+      "theoretical_loss": 4.1714154792080915,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004530452052203518,
+      "loss": 1.6846,
+      "theoretical_loss": 4.165784377879517,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004524541327785133,
+      "loss": 1.6721,
+      "theoretical_loss": 4.160234593519768,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045186306033667483,
+      "loss": 1.7413,
+      "theoretical_loss": 4.15476405777444,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004512719878948364,
+      "loss": 1.7316,
+      "theoretical_loss": 4.149370777139286,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045068091545299793,
+      "loss": 1.6929,
+      "theoretical_loss": 4.144052829464249,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045008984301115945,
+      "loss": 1.6915,
+      "theoretical_loss": 4.138808360656742,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044949877056932097,
+      "loss": 1.6595,
+      "theoretical_loss": 4.133635581570836,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004489076981274825,
+      "loss": 1.6494,
+      "theoretical_loss": 4.128532765070004,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000448316625685644,
+      "loss": 1.6785,
+      "theoretical_loss": 4.123498243252032,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004477255532438056,
+      "loss": 1.6782,
+      "theoretical_loss": 4.118530404825556,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471344808019671,
+      "loss": 1.6705,
+      "theoretical_loss": 4.113627692628464,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044654340836012863,
+      "loss": 1.6777,
+      "theoretical_loss": 4.108788601279149,
+      "tokens_seen": 324403200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5365128506817183,
+      "debugging/distinct-1-grams": 0.7612814402327299,
+      "debugging/distinct-2-grams": 0.9694583753853511,
+      "debugging/entropy-1-grams": 6.003629944255698,
+      "debugging/entropy-2-grams": 7.054987089269872,
+      "debugging/length": 495.25,
+      "debugging/num_segments": 16,
+      "epoch": 0.12,
+      "objective/train/aligned_token_count": 0.5049181580543518,
+      "objective/train/avg_token_score": 0.04383423179388046,
+      "objective/train/docs_used": 197327,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.7676746845245361,
+      "objective/train/original_loss": 4.441669464111328,
+      "objective/train/theoretical_loss": 4.10401167495222,
+      "objective/train/tokens_used": 348140000,
+      "objective/train/unlikelihood_loss": 0.046563390642404556,
+      "theoretical_loss": 4.10401167495222,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044595233591829015,
+      "loss": 1.6963,
+      "theoretical_loss": 4.10401167495222,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044536126347645167,
+      "loss": 1.6701,
+      "theoretical_loss": 4.099295505270921,
+      "tokens_seen": 330956800
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004447701910346132,
+      "loss": 1.6618,
+      "theoretical_loss": 4.094638729309031,
+      "tokens_seen": 334233600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044417911859277476,
+      "loss": 1.6799,
+      "theoretical_loss": 4.090040027695556,
+      "tokens_seen": 337510400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435880461509363,
+      "loss": 1.6659,
+      "theoretical_loss": 4.085498122815992,
+      "tokens_seen": 340787200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004429969737090978,
+      "loss": 1.6489,
+      "theoretical_loss": 4.081011777104333,
+      "tokens_seen": 344064000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004424059012672593,
+      "loss": 1.704,
+      "theoretical_loss": 4.076579791420469,
+      "tokens_seen": 347340800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044181482882542085,
+      "loss": 1.7223,
+      "theoretical_loss": 4.0722010035079155,
+      "tokens_seen": 350617600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044122375638358237,
+      "loss": 1.7051,
+      "theoretical_loss": 4.067874286527197,
+      "tokens_seen": 353894400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004406326839417439,
+      "loss": 1.6944,
+      "theoretical_loss": 4.063598547660519,
+      "tokens_seen": 357171200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044004161149990546,
+      "loss": 1.7215,
+      "theoretical_loss": 4.05937272678363,
+      "tokens_seen": 360448000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000439450539058067,
+      "loss": 1.6765,
+      "theoretical_loss": 4.055195795201069,
+      "tokens_seen": 363724800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388594666162285,
+      "loss": 1.6375,
+      "theoretical_loss": 4.051066754441235,
+      "tokens_seen": 367001600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043826839417439,
+      "loss": 1.6742,
+      "theoretical_loss": 4.04698463510794,
+      "tokens_seen": 370278400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043767732173255155,
+      "loss": 1.6316,
+      "theoretical_loss": 4.042948495785312,
+      "tokens_seen": 373555200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043708624929071307,
+      "loss": 1.6294,
+      "theoretical_loss": 4.038957421993153,
+      "tokens_seen": 376832000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043649517684887464,
+      "loss": 1.6151,
+      "theoretical_loss": 4.035010525189982,
+      "tokens_seen": 380108800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043590410440703616,
+      "loss": 1.6286,
+      "theoretical_loss": 4.031106941821218,
+      "tokens_seen": 383385600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004353130319651977,
+      "loss": 1.6382,
+      "theoretical_loss": 4.027245832410079,
+      "tokens_seen": 386662400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004347219595233592,
+      "loss": 1.5849,
+      "theoretical_loss": 4.023426380688943,
+      "tokens_seen": 389939200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004341308870815207,
+      "loss": 1.5586,
+      "theoretical_loss": 4.019647792769048,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043353981463968225,
+      "loss": 1.5695,
+      "theoretical_loss": 4.015909296346521,
+      "tokens_seen": 396492800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004329605636466806,
+      "loss": 1.5362,
+      "theoretical_loss": 4.012210139942894,
+      "tokens_seen": 399769600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004323694912048421,
+      "loss": 1.5579,
+      "theoretical_loss": 4.008549592178291,
+      "tokens_seen": 403046400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004317784187630036,
+      "loss": 1.5716,
+      "theoretical_loss": 4.004926941075674,
+      "tokens_seen": 406323200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043118734632116514,
+      "loss": 1.5923,
+      "theoretical_loss": 4.001341493394558,
+      "tokens_seen": 409600000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043059627387932666,
+      "loss": 1.5434,
+      "theoretical_loss": 3.997792573992726,
+      "tokens_seen": 412876800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004300052014374882,
+      "loss": 1.533,
+      "theoretical_loss": 3.994279525214554,
+      "tokens_seen": 416153600
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004294141289956497,
+      "loss": 1.549,
+      "theoretical_loss": 3.990801706304647,
+      "tokens_seen": 419430400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004288230565538113,
+      "loss": 1.5335,
+      "theoretical_loss": 3.987358492845532,
+      "tokens_seen": 422707200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004282319841119728,
+      "loss": 1.6157,
+      "theoretical_loss": 3.9839492762182647,
+      "tokens_seen": 425984000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004276409116701343,
+      "loss": 1.585,
+      "theoretical_loss": 3.9805734630848306,
+      "tokens_seen": 429260800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042704983922829584,
+      "loss": 1.5776,
+      "theoretical_loss": 3.9772304748913054,
+      "tokens_seen": 432537600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042645876678645736,
+      "loss": 1.612,
+      "theoretical_loss": 3.973919747390801,
+      "tokens_seen": 435814400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004258676943446189,
+      "loss": 1.597,
+      "theoretical_loss": 3.9706407301852487,
+      "tokens_seen": 439091200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042527662190278045,
+      "loss": 1.5849,
+      "theoretical_loss": 3.9673928862851655,
+      "tokens_seen": 442368000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042468554946094197,
+      "loss": 1.6012,
+      "theoretical_loss": 3.9641756916865463,
+      "tokens_seen": 445644800
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004240944770191035,
+      "loss": 1.5879,
+      "theoretical_loss": 3.960988634964113,
+      "tokens_seen": 448921600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000423503404577265,
+      "loss": 1.5612,
+      "theoretical_loss": 3.9578312168801597,
+      "tokens_seen": 452198400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042291233213542654,
+      "loss": 1.5671,
+      "theoretical_loss": 3.954702950008308,
+      "tokens_seen": 455475200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042232125969358806,
+      "loss": 1.5869,
+      "theoretical_loss": 3.9516033583714734,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042173018725174963,
+      "loss": 1.5735,
+      "theoretical_loss": 3.9485319770934355,
+      "tokens_seen": 462028800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042113911480991115,
+      "loss": 1.576,
+      "theoretical_loss": 3.945488352063391,
+      "tokens_seen": 465305600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042054804236807267,
+      "loss": 1.5879,
+      "theoretical_loss": 3.942472039612926,
+      "tokens_seen": 468582400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199569699262342,
+      "loss": 1.5763,
+      "theoretical_loss": 3.939482606204863,
+      "tokens_seen": 471859200
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041937771893323247,
+      "loss": 1.5946,
+      "theoretical_loss": 3.936519628133466,
+      "tokens_seen": 475136000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.000418786646491394,
+      "loss": 1.6108,
+      "theoretical_loss": 3.9335826912355114,
+      "tokens_seen": 478412800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041819557404955556,
+      "loss": 1.6456,
+      "theoretical_loss": 3.93067139061177,
+      "tokens_seen": 481689600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004176045016077171,
+      "loss": 1.602,
+      "theoretical_loss": 3.927785330358441,
+      "tokens_seen": 484966400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004170134291658786,
+      "loss": 1.6305,
+      "theoretical_loss": 3.9249241233081333,
+      "tokens_seen": 488243200
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/aligned_token_count": 0.5038489699363708,
+      "objective/train/avg_token_score": 0.0046414583921432495,
+      "objective/train/docs_used": 287192,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.8303951025009155,
+      "objective/train/original_loss": 4.303061485290527,
+      "objective/train/theoretical_loss": 3.92208739077998,
+      "objective/train/tokens_used": 511980000,
+      "objective/train/unlikelihood_loss": 0.05593807250261307,
+      "theoretical_loss": 3.92208739077998,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004164223567240401,
+      "loss": 1.5763,
+      "theoretical_loss": 3.92208739077998,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041583128428220164,
+      "loss": 1.5928,
+      "theoretical_loss": 3.919274762338519,
+      "tokens_seen": 494796800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041524021184036317,
+      "loss": 1.6052,
+      "theoretical_loss": 3.9164858755609613,
+      "tokens_seen": 498073600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004146491393985247,
+      "loss": 1.6125,
+      "theoretical_loss": 3.9137203758125176,
+      "tokens_seen": 501350400
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041405806695668626,
+      "loss": 1.5871,
+      "theoretical_loss": 3.910977916029439,
+      "tokens_seen": 504627200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004134669945148478,
+      "loss": 1.6007,
+      "theoretical_loss": 3.908258156509472,
+      "tokens_seen": 507904000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004128759220730093,
+      "loss": 1.6077,
+      "theoretical_loss": 3.905560764709417,
+      "tokens_seen": 511180800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004122848496311708,
+      "loss": 1.5515,
+      "theoretical_loss": 3.9028854150495143,
+      "tokens_seen": 514457600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041169377718933234,
+      "loss": 1.5873,
+      "theoretical_loss": 3.9002317887243834,
+      "tokens_seen": 517734400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041110270474749386,
+      "loss": 1.5527,
+      "theoretical_loss": 3.897599573520247,
+      "tokens_seen": 521011200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041051163230565544,
+      "loss": 1.5625,
+      "theoretical_loss": 3.8949884636382106,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004099205598638169,
+      "loss": 1.49,
+      "theoretical_loss": 3.892398159523345,
+      "tokens_seen": 527564800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040932948742197843,
+      "loss": 1.5195,
+      "theoretical_loss": 3.889828367699349,
+      "tokens_seen": 530841600
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040873841498013995,
+      "loss": 1.4772,
+      "theoretical_loss": 3.8872788006085894,
+      "tokens_seen": 534118400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040814734253830147,
+      "loss": 1.4854,
+      "theoretical_loss": 3.8847491764572926,
+      "tokens_seen": 537395200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000407556270096463,
+      "loss": 1.4966,
+      "theoretical_loss": 3.882239219065708,
+      "tokens_seen": 540672000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040696519765462456,
+      "loss": 1.4861,
+      "theoretical_loss": 3.879748657723039,
+      "tokens_seen": 543948800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004063741252127861,
+      "loss": 1.5319,
+      "theoretical_loss": 3.8772772270469824,
+      "tokens_seen": 547225600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004057830527709476,
+      "loss": 1.5379,
+      "theoretical_loss": 3.8748246668476827,
+      "tokens_seen": 550502400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004051919803291091,
+      "loss": 1.5656,
+      "theoretical_loss": 3.8723907219959486,
+      "tokens_seen": 553779200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040460090788727065,
+      "loss": 1.5567,
+      "theoretical_loss": 3.869975142295573,
+      "tokens_seen": 557056000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040400983544543217,
+      "loss": 1.5761,
+      "theoretical_loss": 3.8675776823595998,
+      "tokens_seen": 560332800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004034187630035937,
+      "loss": 1.5587,
+      "theoretical_loss": 3.8651981014904027,
+      "tokens_seen": 563609600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040282769056175526,
+      "loss": 1.569,
+      "theoretical_loss": 3.8628361635634265,
+      "tokens_seen": 566886400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004022366181199168,
+      "loss": 1.5768,
+      "theoretical_loss": 3.8604916369144666,
+      "tokens_seen": 570163200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004016455456780783,
+      "loss": 1.5213,
+      "theoretical_loss": 3.858164294230354,
+      "tokens_seen": 573440000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004010544732362398,
+      "loss": 1.5503,
+      "theoretical_loss": 3.85585391244293,
+      "tokens_seen": 576716800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040046340079440135,
+      "loss": 1.5641,
+      "theoretical_loss": 3.8535602726261864,
+      "tokens_seen": 579993600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039987232835256287,
+      "loss": 1.5549,
+      "theoretical_loss": 3.851283159896468,
+      "tokens_seen": 583270400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039928125591072444,
+      "loss": 1.5584,
+      "theoretical_loss": 3.8490223633156173,
+      "tokens_seen": 586547200
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039869018346888596,
+      "loss": 1.5083,
+      "theoretical_loss": 3.846777675796974,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003980991110270475,
+      "loss": 1.5641,
+      "theoretical_loss": 3.844548894014116,
+      "tokens_seen": 593100800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000397508038585209,
+      "loss": 1.5583,
+      "theoretical_loss": 3.8423358183122582,
+      "tokens_seen": 596377600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003969169661433705,
+      "loss": 1.5336,
+      "theoretical_loss": 3.840138252622208,
+      "tokens_seen": 599654400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039632589370153205,
+      "loss": 1.5314,
+      "theoretical_loss": 3.837956004376799,
+      "tokens_seen": 602931200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003957348212596936,
+      "loss": 1.5399,
+      "theoretical_loss": 3.8357888844297094,
+      "tokens_seen": 606208000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039514374881785514,
+      "loss": 1.5222,
+      "theoretical_loss": 3.8336367069765958,
+      "tokens_seen": 609484800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039455267637601666,
+      "loss": 1.519,
+      "theoretical_loss": 3.8314992894784536,
+      "tokens_seen": 612761600
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003939616039341782,
+      "loss": 1.5535,
+      "theoretical_loss": 3.829376452587134,
+      "tokens_seen": 616038400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003933705314923397,
+      "loss": 1.5592,
+      "theoretical_loss": 3.827268020072948,
+      "tokens_seen": 619315200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003927794590505012,
+      "loss": 1.5926,
+      "theoretical_loss": 3.8251738187542843,
+      "tokens_seen": 622592000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039218838660866275,
+      "loss": 1.4827,
+      "theoretical_loss": 3.8230936784291787,
+      "tokens_seen": 625868800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003915973141668243,
+      "loss": 1.5234,
+      "theoretical_loss": 3.8210274318087656,
+      "tokens_seen": 629145600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039100624172498584,
+      "loss": 1.505,
+      "theoretical_loss": 3.818974914452557,
+      "tokens_seen": 632422400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039041516928314736,
+      "loss": 1.5051,
+      "theoretical_loss": 3.8169359647054835,
+      "tokens_seen": 635699200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003898240968413089,
+      "loss": 1.5142,
+      "theoretical_loss": 3.8149104236366433,
+      "tokens_seen": 638976000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003892330243994704,
+      "loss": 1.4905,
+      "theoretical_loss": 3.8128981349797098,
+      "tokens_seen": 642252800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003886419519576319,
+      "loss": 1.5262,
+      "theoretical_loss": 3.8108989450749293,
+      "tokens_seen": 645529600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003880508795157935,
+      "loss": 1.5396,
+      "theoretical_loss": 3.8089127028126764,
+      "tokens_seen": 648806400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000387459807073955,
+      "loss": 1.5459,
+      "theoretical_loss": 3.8069392595785083,
+      "tokens_seen": 652083200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5265375629586004,
+      "debugging/distinct-1-grams": 0.7435820408094715,
+      "debugging/distinct-2-grams": 0.9558103821233092,
+      "debugging/entropy-1-grams": 5.931434510687563,
+      "debugging/entropy-2-grams": 6.886416755326388,
+      "debugging/length": 521.9230769230769,
+      "debugging/num_segments": 13,
+      "epoch": 0.23,
+      "objective/train/aligned_token_count": 0.3504704236984253,
+      "objective/train/avg_token_score": 0.022748464718461037,
+      "objective/train/docs_used": 379091,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.335200548171997,
+      "objective/train/original_loss": 4.815954208374023,
+      "objective/train/theoretical_loss": 3.804978469199669,
+      "objective/train/tokens_used": 675820000,
+      "objective/train/unlikelihood_loss": 0.04292531684041023,
+      "theoretical_loss": 3.804978469199669,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038686873463211654,
+      "loss": 1.5604,
+      "theoretical_loss": 3.804978469199669,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038627766219027806,
+      "loss": 1.5451,
+      "theoretical_loss": 3.803030187893005,
+      "tokens_seen": 658636800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003856865897484396,
+      "loss": 1.5325,
+      "theoretical_loss": 3.8010942742142415,
+      "tokens_seen": 661913600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003850955173066011,
+      "loss": 1.5245,
+      "theoretical_loss": 3.799170589008585,
+      "tokens_seen": 665190400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003845044448647627,
+      "loss": 1.5241,
+      "theoretical_loss": 3.7972589953626006,
+      "tokens_seen": 668467200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003839133724229242,
+      "loss": 1.5572,
+      "theoretical_loss": 3.795359358557337,
+      "tokens_seen": 671744000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833222999810857,
+      "loss": 1.5405,
+      "theoretical_loss": 3.79347154602265,
+      "tokens_seen": 675020800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038273122753924724,
+      "loss": 1.5708,
+      "theoretical_loss": 3.7915954272926955,
+      "tokens_seen": 678297600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038214015509740876,
+      "loss": 1.5325,
+      "theoretical_loss": 3.789730873962557,
+      "tokens_seen": 681574400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038154908265557023,
+      "loss": 1.4455,
+      "theoretical_loss": 3.787877759645963,
+      "tokens_seen": 684851200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038095801021373175,
+      "loss": 1.492,
+      "theoretical_loss": 3.7860359599340776,
+      "tokens_seen": 688128000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003803669377718933,
+      "loss": 1.5952,
+      "theoretical_loss": 3.784205352355321,
+      "tokens_seen": 691404800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037977586533005485,
+      "loss": 1.5912,
+      "theoretical_loss": 3.782385816336189,
+      "tokens_seen": 694681600
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037918479288821637,
+      "loss": 1.5781,
+      "theoretical_loss": 3.7805772331630516,
+      "tokens_seen": 697958400
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003785937204463779,
+      "loss": 1.5328,
+      "theoretical_loss": 3.7787794859448898,
+      "tokens_seen": 701235200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003780026480045394,
+      "loss": 1.5311,
+      "theoretical_loss": 3.7769924595769546,
+      "tokens_seen": 704512000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037741157556270093,
+      "loss": 1.5117,
+      "theoretical_loss": 3.7752160407053115,
+      "tokens_seen": 707788800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003768205031208625,
+      "loss": 1.5205,
+      "theoretical_loss": 3.7734501176922493,
+      "tokens_seen": 711065600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000376229430679024,
+      "loss": 1.5291,
+      "theoretical_loss": 3.7716945805825337,
+      "tokens_seen": 714342400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037563835823718555,
+      "loss": 1.5385,
+      "theoretical_loss": 3.7699493210704667,
+      "tokens_seen": 717619200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037504728579534707,
+      "loss": 1.4824,
+      "theoretical_loss": 3.7682142324677455,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003744562133535086,
+      "loss": 1.5445,
+      "theoretical_loss": 3.7664892096720886,
+      "tokens_seen": 724172800
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003738651409116701,
+      "loss": 1.5027,
+      "theoretical_loss": 3.7647741491366067,
+      "tokens_seen": 727449600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003732740684698317,
+      "loss": 1.5255,
+      "theoretical_loss": 3.7630689488399027,
+      "tokens_seen": 730726400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003726829960279932,
+      "loss": 1.5109,
+      "theoretical_loss": 3.7613735082568764,
+      "tokens_seen": 734003200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003720919235861547,
+      "loss": 1.5496,
+      "theoretical_loss": 3.759687728330217,
+      "tokens_seen": 737280000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037150085114431625,
+      "loss": 1.5389,
+      "theoretical_loss": 3.75801151144256,
+      "tokens_seen": 740556800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003709216001513145,
+      "loss": 1.5425,
+      "theoretical_loss": 3.756344761389295,
+      "tokens_seen": 743833600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037033052770947604,
+      "loss": 1.5185,
+      "theoretical_loss": 3.754687383352003,
+      "tokens_seen": 747110400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003697394552676376,
+      "loss": 1.5102,
+      "theoretical_loss": 3.7530392838725097,
+      "tokens_seen": 750387200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036914838282579913,
+      "loss": 1.4951,
+      "theoretical_loss": 3.751400370827529,
+      "tokens_seen": 753664000
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036855731038396065,
+      "loss": 1.4927,
+      "theoretical_loss": 3.749770553403895,
+      "tokens_seen": 756940800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679662379421222,
+      "loss": 1.4834,
+      "theoretical_loss": 3.748149742074355,
+      "tokens_seen": 760217600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003673751655002837,
+      "loss": 1.5153,
+      "theoretical_loss": 3.746537848573908,
+      "tokens_seen": 763494400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003667840930584452,
+      "loss": 1.4942,
+      "theoretical_loss": 3.744934785876686,
+      "tokens_seen": 766771200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036619302061660674,
+      "loss": 1.5063,
+      "theoretical_loss": 3.7433404681733475,
+      "tokens_seen": 770048000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003656019481747683,
+      "loss": 1.4933,
+      "theoretical_loss": 3.7417548108489846,
+      "tokens_seen": 773324800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036501087573292983,
+      "loss": 1.506,
+      "theoretical_loss": 3.740177730461517,
+      "tokens_seen": 776601600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036441980329109135,
+      "loss": 1.5283,
+      "theoretical_loss": 3.73860914472057,
+      "tokens_seen": 779878400
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003638287308492529,
+      "loss": 1.4958,
+      "theoretical_loss": 3.7370489724668197,
+      "tokens_seen": 783155200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003632376584074144,
+      "loss": 1.5108,
+      "theoretical_loss": 3.735497133651788,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003626465859655759,
+      "loss": 1.538,
+      "theoretical_loss": 3.733953549318091,
+      "tokens_seen": 789708800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003620555135237375,
+      "loss": 1.5356,
+      "theoretical_loss": 3.7324181415801094,
+      "tokens_seen": 792985600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000361464441081899,
+      "loss": 1.5407,
+      "theoretical_loss": 3.7308908336050814,
+      "tokens_seen": 796262400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036087336864006053,
+      "loss": 1.5173,
+      "theoretical_loss": 3.729371549594614,
+      "tokens_seen": 799539200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036028229619822205,
+      "loss": 1.5213,
+      "theoretical_loss": 3.7278602147665776,
+      "tokens_seen": 802816000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003596912237563836,
+      "loss": 1.5136,
+      "theoretical_loss": 3.726356755337407,
+      "tokens_seen": 806092800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003591001513145451,
+      "loss": 1.47,
+      "theoretical_loss": 3.724861098504767,
+      "tokens_seen": 809369600
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035850907887270667,
+      "loss": 1.5481,
+      "theoretical_loss": 3.7233731724305974,
+      "tokens_seen": 812646400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003579180064308682,
+      "loss": 1.5137,
+      "theoretical_loss": 3.7218929062245105,
+      "tokens_seen": 815923200
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/aligned_token_count": 0.4432734549045563,
+      "objective/train/avg_token_score": 0.009074240922927856,
+      "objective/train/docs_used": 471128,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.5172392129898071,
+      "objective/train/original_loss": 4.30383825302124,
+      "objective/train/theoretical_loss": 3.7204202299275475,
+      "objective/train/tokens_used": 839660000,
+      "objective/train/unlikelihood_loss": 0.0707290768623352,
+      "theoretical_loss": 3.7204202299275475,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003573269339890297,
+      "loss": 1.5079,
+      "theoretical_loss": 3.7204202299275475,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035673586154719123,
+      "loss": 1.5334,
+      "theoretical_loss": 3.7189550744962707,
+      "tokens_seen": 822476800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035614478910535275,
+      "loss": 1.5135,
+      "theoretical_loss": 3.717497371787192,
+      "tokens_seen": 825753600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003555537166635143,
+      "loss": 1.5284,
+      "theoretical_loss": 3.7160470545415274,
+      "tokens_seen": 829030400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003549626442216758,
+      "loss": 1.4813,
+      "theoretical_loss": 3.714604056370267,
+      "tokens_seen": 832307200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035437157177983737,
+      "loss": 1.4847,
+      "theoretical_loss": 3.713168311739558,
+      "tokens_seen": 835584000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003537804993379989,
+      "loss": 1.5102,
+      "theoretical_loss": 3.7117397559563843,
+      "tokens_seen": 838860800
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003531894268961604,
+      "loss": 1.4839,
+      "theoretical_loss": 3.710318325154545,
+      "tokens_seen": 842137600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035259835445432193,
+      "loss": 1.5297,
+      "theoretical_loss": 3.7089039562809223,
+      "tokens_seen": 845414400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035200728201248345,
+      "loss": 1.5091,
+      "theoretical_loss": 3.7074965870820193,
+      "tokens_seen": 848691200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000351416209570645,
+      "loss": 1.4913,
+      "theoretical_loss": 3.7060961560907857,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035082513712880655,
+      "loss": 1.5366,
+      "theoretical_loss": 3.7047026026137,
+      "tokens_seen": 855244800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035023406468696807,
+      "loss": 1.4965,
+      "theoretical_loss": 3.7033158667181154,
+      "tokens_seen": 858521600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003496429922451296,
+      "loss": 1.5212,
+      "theoretical_loss": 3.701935889219863,
+      "tokens_seen": 861798400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003490519198032911,
+      "loss": 1.5212,
+      "theoretical_loss": 3.7005626116710966,
+      "tokens_seen": 865075200
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034846084736145263,
+      "loss": 1.5229,
+      "theoretical_loss": 3.69919597634839,
+      "tokens_seen": 868352000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034786977491961415,
+      "loss": 1.5199,
+      "theoretical_loss": 3.6978359262410603,
+      "tokens_seen": 871628800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034727870247777573,
+      "loss": 1.5108,
+      "theoretical_loss": 3.6964824050397276,
+      "tokens_seen": 874905600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.000346699451484774,
+      "loss": 1.4647,
+      "theoretical_loss": 3.6951353571251015,
+      "tokens_seen": 878182400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003461083790429355,
+      "loss": 1.4852,
+      "theoretical_loss": 3.693794727556988,
+      "tokens_seen": 881459200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034551730660109704,
+      "loss": 1.4921,
+      "theoretical_loss": 3.692460462063506,
+      "tokens_seen": 884736000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034492623415925856,
+      "loss": 1.5273,
+      "theoretical_loss": 3.691132507030521,
+      "tokens_seen": 888012800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003443351617174201,
+      "loss": 1.5029,
+      "theoretical_loss": 3.6898108094912816,
+      "tokens_seen": 891289600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003437440892755816,
+      "loss": 1.4614,
+      "theoretical_loss": 3.6884953171162556,
+      "tokens_seen": 894566400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003431530168337432,
+      "loss": 1.474,
+      "theoretical_loss": 3.6871859782031624,
+      "tokens_seen": 897843200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003425619443919047,
+      "loss": 1.4795,
+      "theoretical_loss": 3.685882741667202,
+      "tokens_seen": 901120000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003419708719500662,
+      "loss": 1.4727,
+      "theoretical_loss": 3.684585557031461,
+      "tokens_seen": 904396800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034137979950822774,
+      "loss": 1.4769,
+      "theoretical_loss": 3.6832943744175126,
+      "tokens_seen": 907673600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034078872706638926,
+      "loss": 1.5173,
+      "theoretical_loss": 3.682009144536188,
+      "tokens_seen": 910950400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003401976546245508,
+      "loss": 1.5193,
+      "theoretical_loss": 3.680729818678526,
+      "tokens_seen": 914227200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033960658218271236,
+      "loss": 1.4737,
+      "theoretical_loss": 3.6794563487068936,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003390155097408739,
+      "loss": 1.4972,
+      "theoretical_loss": 3.6781886870462692,
+      "tokens_seen": 920780800
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003384244372990354,
+      "loss": 1.5335,
+      "theoretical_loss": 3.676926786675698,
+      "tokens_seen": 924057600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003378333648571969,
+      "loss": 1.5096,
+      "theoretical_loss": 3.6756706011198963,
+      "tokens_seen": 927334400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033724229241535844,
+      "loss": 1.483,
+      "theoretical_loss": 3.6744200844410217,
+      "tokens_seen": 930611200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033665121997351996,
+      "loss": 1.4676,
+      "theoretical_loss": 3.6731751912305914,
+      "tokens_seen": 933888000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033606014753168154,
+      "loss": 1.4933,
+      "theoretical_loss": 3.671935876601547,
+      "tokens_seen": 937164800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033546907508984306,
+      "loss": 1.4452,
+      "theoretical_loss": 3.6707020961804715,
+      "tokens_seen": 940441600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003348780026480046,
+      "loss": 1.4928,
+      "theoretical_loss": 3.6694738060999468,
+      "tokens_seen": 943718400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003342869302061661,
+      "loss": 1.4841,
+      "theoretical_loss": 3.668250962991049,
+      "tokens_seen": 946995200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033369585776432757,
+      "loss": 1.4887,
+      "theoretical_loss": 3.667033523975983,
+      "tokens_seen": 950272000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003331047853224891,
+      "loss": 1.5141,
+      "theoretical_loss": 3.66582144666085,
+      "tokens_seen": 953548800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003325137128806506,
+      "loss": 1.4972,
+      "theoretical_loss": 3.664614689128546,
+      "tokens_seen": 956825600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003319226404388122,
+      "loss": 1.4986,
+      "theoretical_loss": 3.6634132099317886,
+      "tokens_seen": 960102400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003313315679969737,
+      "loss": 1.461,
+      "theoretical_loss": 3.662216968086267,
+      "tokens_seen": 963379200
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003307404955551352,
+      "loss": 1.4882,
+      "theoretical_loss": 3.6610259230639217,
+      "tokens_seen": 966656000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033014942311329675,
+      "loss": 1.4241,
+      "theoretical_loss": 3.659840034786333,
+      "tokens_seen": 969932800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032955835067145827,
+      "loss": 1.4232,
+      "theoretical_loss": 3.6586592636182376,
+      "tokens_seen": 973209600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003289672782296198,
+      "loss": 1.4193,
+      "theoretical_loss": 3.6574835703611566,
+      "tokens_seen": 976486400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032837620578778136,
+      "loss": 1.4464,
+      "theoretical_loss": 3.6563129162471313,
+      "tokens_seen": 979763200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4286046663919377,
+      "debugging/distinct-1-grams": 0.8147567798871364,
+      "debugging/distinct-2-grams": 0.9823269374342457,
+      "debugging/entropy-1-grams": 6.1671920556004824,
+      "debugging/entropy-2-grams": 6.947028138756313,
+      "debugging/length": 477.53333333333336,
+      "debugging/num_segments": 15,
+      "epoch": 0.35,
+      "objective/train/aligned_token_count": 0.41187071800231934,
+      "objective/train/avg_token_score": 0.020625250414013863,
+      "objective/train/docs_used": 560408,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.4003713130950928,
+      "objective/train/original_loss": 4.378409385681152,
+      "objective/train/theoretical_loss": 3.6551472629325787,
+      "objective/train/tokens_used": 1003500000,
+      "objective/train/unlikelihood_loss": 0.060329046100378036,
+      "theoretical_loss": 3.6551472629325787,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003277851333459429,
+      "loss": 1.4406,
+      "theoretical_loss": 3.6551472629325787,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003271940609041044,
+      "loss": 1.4229,
+      "theoretical_loss": 3.653986572492247,
+      "tokens_seen": 986316800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003266029884622659,
+      "loss": 1.4242,
+      "theoretical_loss": 3.65283080741328,
+      "tokens_seen": 989593600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032601191602042745,
+      "loss": 1.487,
+      "theoretical_loss": 3.6516799305893866,
+      "tokens_seen": 992870400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032542084357858897,
+      "loss": 1.4475,
+      "theoretical_loss": 3.6505339053151076,
+      "tokens_seen": 996147200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032482977113675054,
+      "loss": 1.4483,
+      "theoretical_loss": 3.649392695280186,
+      "tokens_seen": 999424000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032423869869491206,
+      "loss": 1.4514,
+      "theoretical_loss": 3.6482562645640337,
+      "tokens_seen": 1002700800
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003236476262530736,
+      "loss": 1.4288,
+      "theoretical_loss": 3.6471245776302883,
+      "tokens_seen": 1005977600
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003230565538112351,
+      "loss": 1.4819,
+      "theoretical_loss": 3.6459975993214724,
+      "tokens_seen": 1009254400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003224773028182334,
+      "loss": 1.4347,
+      "theoretical_loss": 3.6448752948537377,
+      "tokens_seen": 1012531200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003218862303763949,
+      "loss": 1.4786,
+      "theoretical_loss": 3.6437576298116996,
+      "tokens_seen": 1015808000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032129515793455647,
+      "loss": 1.4922,
+      "theoretical_loss": 3.6426445701433607,
+      "tokens_seen": 1019084800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000320704085492718,
+      "loss": 1.4672,
+      "theoretical_loss": 3.6415360821551226,
+      "tokens_seen": 1022361600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003201130130508795,
+      "loss": 1.474,
+      "theoretical_loss": 3.6404321325068754,
+      "tokens_seen": 1025638400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031952194060904103,
+      "loss": 1.4747,
+      "theoretical_loss": 3.639332688207178,
+      "tokens_seen": 1028915200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031893086816720255,
+      "loss": 1.4964,
+      "theoretical_loss": 3.6382377166085096,
+      "tokens_seen": 1032192000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003183397957253641,
+      "loss": 1.5124,
+      "theoretical_loss": 3.6371471854026147,
+      "tokens_seen": 1035468800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003177487232835256,
+      "loss": 1.5198,
+      "theoretical_loss": 3.6360610626159087,
+      "tokens_seen": 1038745600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031715765084168717,
+      "loss": 1.4836,
+      "theoretical_loss": 3.634979316604973,
+      "tokens_seen": 1042022400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003165665783998487,
+      "loss": 1.4681,
+      "theoretical_loss": 3.6339019160521198,
+      "tokens_seen": 1045299200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003159755059580102,
+      "loss": 1.4937,
+      "theoretical_loss": 3.632828829961029,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031538443351617173,
+      "loss": 1.4897,
+      "theoretical_loss": 3.631760027652461,
+      "tokens_seen": 1051852800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031479336107433325,
+      "loss": 1.5408,
+      "theoretical_loss": 3.630695478760034,
+      "tokens_seen": 1055129600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003142022886324948,
+      "loss": 1.5123,
+      "theoretical_loss": 3.6296351532260767,
+      "tokens_seen": 1058406400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031361121619065635,
+      "loss": 1.5042,
+      "theoretical_loss": 3.6285790212975435,
+      "tokens_seen": 1061683200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031302014374881787,
+      "loss": 1.5175,
+      "theoretical_loss": 3.6275270535220008,
+      "tokens_seen": 1064960000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003124290713069794,
+      "loss": 1.4928,
+      "theoretical_loss": 3.626479220743673,
+      "tokens_seen": 1068236800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003118379988651409,
+      "loss": 1.4871,
+      "theoretical_loss": 3.6254354940995586,
+      "tokens_seen": 1071513600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031124692642330243,
+      "loss": 1.4979,
+      "theoretical_loss": 3.624395845015602,
+      "tokens_seen": 1074790400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031065585398146395,
+      "loss": 1.4906,
+      "theoretical_loss": 3.6233602452029348,
+      "tokens_seen": 1078067200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031006478153962553,
+      "loss": 1.5379,
+      "theoretical_loss": 3.6223286666541683,
+      "tokens_seen": 1081344000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030947370909778705,
+      "loss": 1.5138,
+      "theoretical_loss": 3.621301081639753,
+      "tokens_seen": 1084620800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030888263665594857,
+      "loss": 1.518,
+      "theoretical_loss": 3.6202774627043923,
+      "tokens_seen": 1087897600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003082915642141101,
+      "loss": 1.5085,
+      "theoretical_loss": 3.619257782663513,
+      "tokens_seen": 1091174400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003077004917722716,
+      "loss": 1.5167,
+      "theoretical_loss": 3.618242014599793,
+      "tokens_seen": 1094451200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030710941933043313,
+      "loss": 1.5158,
+      "theoretical_loss": 3.617230131859743,
+      "tokens_seen": 1097728000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030651834688859465,
+      "loss": 1.5032,
+      "theoretical_loss": 3.6162221080503416,
+      "tokens_seen": 1101004800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030592727444675623,
+      "loss": 1.4688,
+      "theoretical_loss": 3.615217917035726,
+      "tokens_seen": 1104281600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030533620200491775,
+      "loss": 1.4817,
+      "theoretical_loss": 3.614217532933929,
+      "tokens_seen": 1107558400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030474512956307927,
+      "loss": 1.4873,
+      "theoretical_loss": 3.6132209301136715,
+      "tokens_seen": 1110835200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003041540571212408,
+      "loss": 1.4963,
+      "theoretical_loss": 3.612228083191205,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003035629846794023,
+      "loss": 1.5406,
+      "theoretical_loss": 3.611238967027199,
+      "tokens_seen": 1117388800
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030297191223756383,
+      "loss": 1.4879,
+      "theoretical_loss": 3.610253556723679,
+      "tokens_seen": 1120665600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003023808397957254,
+      "loss": 1.4971,
+      "theoretical_loss": 3.609271827621014,
+      "tokens_seen": 1123942400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030178976735388693,
+      "loss": 1.5014,
+      "theoretical_loss": 3.6082937552949463,
+      "tokens_seen": 1127219200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030119869491204845,
+      "loss": 1.4862,
+      "theoretical_loss": 3.607319315553669,
+      "tokens_seen": 1130496000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030060762247020997,
+      "loss": 1.4379,
+      "theoretical_loss": 3.6063484844349456,
+      "tokens_seen": 1133772800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003000165500283715,
+      "loss": 1.4522,
+      "theoretical_loss": 3.605381238203279,
+      "tokens_seen": 1137049600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000299425477586533,
+      "loss": 1.4868,
+      "theoretical_loss": 3.604417553347117,
+      "tokens_seen": 1140326400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002988344051446946,
+      "loss": 1.4577,
+      "theoretical_loss": 3.603457406576106,
+      "tokens_seen": 1143603200
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/aligned_token_count": 0.40093475580215454,
+      "objective/train/avg_token_score": 0.022879082709550858,
+      "objective/train/docs_used": 649861,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.3152453899383545,
+      "objective/train/original_loss": 4.717131614685059,
+      "objective/train/theoretical_loss": 3.602500774818379,
+      "objective/train/tokens_used": 1167340000,
+      "objective/train/unlikelihood_loss": 0.05485496297478676,
+      "theoretical_loss": 3.602500774818379,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029825515415169286,
+      "loss": 1.4531,
+      "theoretical_loss": 3.602500774818379,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002976640817098544,
+      "loss": 1.5139,
+      "theoretical_loss": 3.601547635217892,
+      "tokens_seen": 1150156800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002970730092680159,
+      "loss": 1.4711,
+      "theoretical_loss": 3.6005979651317976,
+      "tokens_seen": 1153433600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002964819368261774,
+      "loss": 1.4994,
+      "theoretical_loss": 3.599651742127855,
+      "tokens_seen": 1156710400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029589086438433894,
+      "loss": 1.5085,
+      "theoretical_loss": 3.5987089439818805,
+      "tokens_seen": 1159987200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002952997919425005,
+      "loss": 1.5031,
+      "theoretical_loss": 3.5977695486752426,
+      "tokens_seen": 1163264000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029470871950066204,
+      "loss": 1.4878,
+      "theoretical_loss": 3.596833534392379,
+      "tokens_seen": 1166540800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029411764705882356,
+      "loss": 1.4909,
+      "theoretical_loss": 3.595900879518368,
+      "tokens_seen": 1169817600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002935265746169851,
+      "loss": 1.4758,
+      "theoretical_loss": 3.594971562636521,
+      "tokens_seen": 1173094400
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002929355021751466,
+      "loss": 1.439,
+      "theoretical_loss": 3.5940455625260226,
+      "tokens_seen": 1176371200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002923444297333081,
+      "loss": 1.4844,
+      "theoretical_loss": 3.5931228581595938,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029175335729146964,
+      "loss": 1.4677,
+      "theoretical_loss": 3.5922034287011995,
+      "tokens_seen": 1182924800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002911622848496312,
+      "loss": 1.4536,
+      "theoretical_loss": 3.5912872535037828,
+      "tokens_seen": 1186201600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029057121240779274,
+      "loss": 1.5246,
+      "theoretical_loss": 3.590374312107035,
+      "tokens_seen": 1189478400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028998013996595426,
+      "loss": 1.5373,
+      "theoretical_loss": 3.5894645842351993,
+      "tokens_seen": 1192755200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002893890675241158,
+      "loss": 1.5291,
+      "theoretical_loss": 3.588558049794902,
+      "tokens_seen": 1196032000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002887979950822773,
+      "loss": 1.4817,
+      "theoretical_loss": 3.5876546888730187,
+      "tokens_seen": 1199308800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002882069226404388,
+      "loss": 1.4989,
+      "theoretical_loss": 3.5867544817345713,
+      "tokens_seen": 1202585600
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002876158501986004,
+      "loss": 1.5032,
+      "theoretical_loss": 3.585857408820652,
+      "tokens_seen": 1205862400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002870247777567619,
+      "loss": 1.502,
+      "theoretical_loss": 3.58496345074638,
+      "tokens_seen": 1209139200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028643370531492344,
+      "loss": 1.5109,
+      "theoretical_loss": 3.5840725882988873,
+      "tokens_seen": 1212416000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002858426328730849,
+      "loss": 1.4436,
+      "theoretical_loss": 3.5831848024353317,
+      "tokens_seen": 1215692800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002852515604312464,
+      "loss": 1.4737,
+      "theoretical_loss": 3.5823000742809374,
+      "tokens_seen": 1218969600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028466048798940794,
+      "loss": 1.4855,
+      "theoretical_loss": 3.5814183851270673,
+      "tokens_seen": 1222246400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002840694155475695,
+      "loss": 1.4717,
+      "theoretical_loss": 3.5805397164293167,
+      "tokens_seen": 1225523200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028347834310573104,
+      "loss": 1.4701,
+      "theoretical_loss": 3.5796640498056407,
+      "tokens_seen": 1228800000
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028288727066389256,
+      "loss": 1.4853,
+      "theoretical_loss": 3.5787913670345013,
+      "tokens_seen": 1232076800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002822961982220541,
+      "loss": 1.4906,
+      "theoretical_loss": 3.577921650053045,
+      "tokens_seen": 1235353600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002817051257802156,
+      "loss": 1.5013,
+      "theoretical_loss": 3.577054880955303,
+      "tokens_seen": 1238630400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002811140533383771,
+      "loss": 1.4887,
+      "theoretical_loss": 3.5761910419904193,
+      "tokens_seen": 1241907200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028052298089653864,
+      "loss": 1.5223,
+      "theoretical_loss": 3.5753301155609014,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002799319084547002,
+      "loss": 1.4843,
+      "theoretical_loss": 3.574472084220896,
+      "tokens_seen": 1248460800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027934083601286174,
+      "loss": 1.4778,
+      "theoretical_loss": 3.5736169306744885,
+      "tokens_seen": 1251737600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027874976357102326,
+      "loss": 1.45,
+      "theoretical_loss": 3.572764637774024,
+      "tokens_seen": 1255014400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002781586911291848,
+      "loss": 1.4591,
+      "theoretical_loss": 3.571915188518457,
+      "tokens_seen": 1258291200
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002775676186873463,
+      "loss": 1.4626,
+      "theoretical_loss": 3.571068566051716,
+      "tokens_seen": 1261568000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002769765462455078,
+      "loss": 1.5087,
+      "theoretical_loss": 3.5702247536610976,
+      "tokens_seen": 1264844800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002763854738036694,
+      "loss": 1.5145,
+      "theoretical_loss": 3.5693837347756783,
+      "tokens_seen": 1268121600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002757944013618309,
+      "loss": 1.446,
+      "theoretical_loss": 3.5685454929647475,
+      "tokens_seen": 1271398400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027520332891999244,
+      "loss": 1.478,
+      "theoretical_loss": 3.5677100119362675,
+      "tokens_seen": 1274675200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027461225647815396,
+      "loss": 1.4301,
+      "theoretical_loss": 3.566877275535345,
+      "tokens_seen": 1277952000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027403300548515223,
+      "loss": 1.4381,
+      "theoretical_loss": 3.566047267742733,
+      "tokens_seen": 1281228800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027344193304331375,
+      "loss": 1.4561,
+      "theoretical_loss": 3.5652199726733453,
+      "tokens_seen": 1284505600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027285086060147533,
+      "loss": 1.4529,
+      "theoretical_loss": 3.564395374574796,
+      "tokens_seen": 1287782400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027225978815963685,
+      "loss": 1.4765,
+      "theoretical_loss": 3.5635734578259557,
+      "tokens_seen": 1291059200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027166871571779837,
+      "loss": 1.4744,
+      "theoretical_loss": 3.5627542069355282,
+      "tokens_seen": 1294336000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002710776432759599,
+      "loss": 1.4313,
+      "theoretical_loss": 3.5619376065406474,
+      "tokens_seen": 1297612800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002704865708341214,
+      "loss": 1.4544,
+      "theoretical_loss": 3.5611236414054868,
+      "tokens_seen": 1300889600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026989549839228293,
+      "loss": 1.4612,
+      "theoretical_loss": 3.560312296419899,
+      "tokens_seen": 1304166400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026930442595044445,
+      "loss": 1.4676,
+      "theoretical_loss": 3.55950355659806,
+      "tokens_seen": 1307443200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.49020908264157476,
+      "debugging/distinct-1-grams": 0.768901113497886,
+      "debugging/distinct-2-grams": 0.9428782333551957,
+      "debugging/entropy-1-grams": 6.085999550681761,
+      "debugging/entropy-2-grams": 7.0033060167714964,
+      "debugging/length": 490.2352941176471,
+      "debugging/num_segments": 17,
+      "epoch": 0.47,
+      "objective/train/aligned_token_count": 0.4558895528316498,
+      "objective/train/avg_token_score": 0.020577482879161835,
+      "objective/train/docs_used": 741674,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.4862196445465088,
+      "objective/train/original_loss": 4.171820163726807,
+      "objective/train/theoretical_loss": 3.558697407077142,
+      "objective/train/tokens_used": 1331180000,
+      "objective/train/unlikelihood_loss": 0.06883184611797333,
+      "theoretical_loss": 3.558697407077142,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026871335350860603,
+      "loss": 1.4542,
+      "theoretical_loss": 3.558697407077142,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026812228106676755,
+      "loss": 1.4879,
+      "theoretical_loss": 3.5578938331159975,
+      "tokens_seen": 1313996800
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026753120862492907,
+      "loss": 1.4622,
+      "theoretical_loss": 3.557092820093863,
+      "tokens_seen": 1317273600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002669401361830906,
+      "loss": 1.5093,
+      "theoretical_loss": 3.556294353509079,
+      "tokens_seen": 1320550400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002663490637412521,
+      "loss": 1.4793,
+      "theoretical_loss": 3.555498418977828,
+      "tokens_seen": 1323827200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026575799129941363,
+      "loss": 1.463,
+      "theoretical_loss": 3.5547050022328874,
+      "tokens_seen": 1327104000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002651669188575752,
+      "loss": 1.4461,
+      "theoretical_loss": 3.553914089122399,
+      "tokens_seen": 1330380800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026457584641573673,
+      "loss": 1.4654,
+      "theoretical_loss": 3.553125665608655,
+      "tokens_seen": 1333657600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026398477397389825,
+      "loss": 1.4441,
+      "theoretical_loss": 3.5523397177669005,
+      "tokens_seen": 1336934400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026339370153205977,
+      "loss": 1.4948,
+      "theoretical_loss": 3.551556231784149,
+      "tokens_seen": 1340211200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002628026290902213,
+      "loss": 1.4395,
+      "theoretical_loss": 3.5507751939580148,
+      "tokens_seen": 1343488000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002622115566483828,
+      "loss": 1.4639,
+      "theoretical_loss": 3.5499965906955606,
+      "tokens_seen": 1346764800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002616204842065444,
+      "loss": 1.487,
+      "theoretical_loss": 3.549220408512161,
+      "tokens_seen": 1350041600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002610294117647059,
+      "loss": 1.4795,
+      "theoretical_loss": 3.5484466340303755,
+      "tokens_seen": 1353318400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026043833932286743,
+      "loss": 1.4466,
+      "theoretical_loss": 3.547675253978843,
+      "tokens_seen": 1356595200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025984726688102895,
+      "loss": 1.5098,
+      "theoretical_loss": 3.5469062551911854,
+      "tokens_seen": 1359872000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025925619443919047,
+      "loss": 1.5321,
+      "theoretical_loss": 3.5461396246049244,
+      "tokens_seen": 1363148800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.000258665121997352,
+      "loss": 1.5263,
+      "theoretical_loss": 3.545375349260419,
+      "tokens_seen": 1366425600
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002580740495555135,
+      "loss": 1.5072,
+      "theoretical_loss": 3.544613416299808,
+      "tokens_seen": 1369702400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002574829771136751,
+      "loss": 1.4847,
+      "theoretical_loss": 3.5438538129659687,
+      "tokens_seen": 1372979200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002568919046718366,
+      "loss": 1.4943,
+      "theoretical_loss": 3.5430965266014933,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025630083222999813,
+      "loss": 1.4781,
+      "theoretical_loss": 3.5423415446476705,
+      "tokens_seen": 1379532800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025570975978815965,
+      "loss": 1.4437,
+      "theoretical_loss": 3.541588854643487,
+      "tokens_seen": 1382809600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025511868734632117,
+      "loss": 1.4702,
+      "theoretical_loss": 3.5408384442246343,
+      "tokens_seen": 1386086400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002545276149044827,
+      "loss": 1.4422,
+      "theoretical_loss": 3.540090301122535,
+      "tokens_seen": 1389363200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025393654246264426,
+      "loss": 1.4321,
+      "theoretical_loss": 3.5393444131633762,
+      "tokens_seen": 1392640000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002533454700208058,
+      "loss": 1.4792,
+      "theoretical_loss": 3.5386007682671576,
+      "tokens_seen": 1395916800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002527543975789673,
+      "loss": 1.4551,
+      "theoretical_loss": 3.5378593544467494,
+      "tokens_seen": 1399193600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025216332513712883,
+      "loss": 1.4662,
+      "theoretical_loss": 3.537120159806962,
+      "tokens_seen": 1402470400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025157225269529035,
+      "loss": 1.4785,
+      "theoretical_loss": 3.5363831725436317,
+      "tokens_seen": 1405747200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025098118025345187,
+      "loss": 1.4752,
+      "theoretical_loss": 3.5356483809427104,
+      "tokens_seen": 1409024000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025039010781161344,
+      "loss": 1.4628,
+      "theoretical_loss": 3.53491577337937,
+      "tokens_seen": 1412300800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002497990353697749,
+      "loss": 1.4567,
+      "theoretical_loss": 3.5341853383171205,
+      "tokens_seen": 1415577600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002492079629279365,
+      "loss": 1.4643,
+      "theoretical_loss": 3.5334570643069325,
+      "tokens_seen": 1418854400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000248616890486098,
+      "loss": 1.457,
+      "theoretical_loss": 3.5327309399863767,
+      "tokens_seen": 1422131200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024802581804425947,
+      "loss": 1.4801,
+      "theoretical_loss": 3.5320069540787706,
+      "tokens_seen": 1425408000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024743474560242105,
+      "loss": 1.4674,
+      "theoretical_loss": 3.531285095392335,
+      "tokens_seen": 1428684800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024684367316058257,
+      "loss": 1.4647,
+      "theoretical_loss": 3.5305653528193623,
+      "tokens_seen": 1431961600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002462526007187441,
+      "loss": 1.4605,
+      "theoretical_loss": 3.5298477153353955,
+      "tokens_seen": 1435238400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002456733497257424,
+      "loss": 1.4482,
+      "theoretical_loss": 3.5291321719984134,
+      "tokens_seen": 1438515200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024508227728390394,
+      "loss": 1.4702,
+      "theoretical_loss": 3.5284187119480315,
+      "tokens_seen": 1441792000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024449120484206546,
+      "loss": 1.4459,
+      "theoretical_loss": 3.527707324404704,
+      "tokens_seen": 1445068800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024390013240022698,
+      "loss": 1.4525,
+      "theoretical_loss": 3.5269979986689464,
+      "tokens_seen": 1448345600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002433090599583885,
+      "loss": 1.4723,
+      "theoretical_loss": 3.5262907241205568,
+      "tokens_seen": 1451622400
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024271798751655002,
+      "loss": 1.4657,
+      "theoretical_loss": 3.5255854902178534,
+      "tokens_seen": 1454899200
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024212691507471157,
+      "loss": 1.4391,
+      "theoretical_loss": 3.524882286496917,
+      "tokens_seen": 1458176000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002415358426328731,
+      "loss": 1.4652,
+      "theoretical_loss": 3.524181102570846,
+      "tokens_seen": 1461452800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002409447701910346,
+      "loss": 1.4422,
+      "theoretical_loss": 3.5234819281290166,
+      "tokens_seen": 1464729600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024035369774919616,
+      "loss": 1.4713,
+      "theoretical_loss": 3.5227847529363547,
+      "tokens_seen": 1468006400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023976262530735768,
+      "loss": 1.4807,
+      "theoretical_loss": 3.522089566832615,
+      "tokens_seen": 1471283200
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/aligned_token_count": 0.3960471749305725,
+      "objective/train/avg_token_score": 0.030484912917017937,
+      "objective/train/docs_used": 835915,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.3865476846694946,
+      "objective/train/original_loss": 4.778916358947754,
+      "objective/train/theoretical_loss": 3.521396359731668,
+      "objective/train/tokens_used": 1495020000,
+      "objective/train/unlikelihood_loss": 0.047592658549547195,
+      "theoretical_loss": 3.521396359731668,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002391715528655192,
+      "loss": 1.4471,
+      "theoretical_loss": 3.521396359731668,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023858048042368072,
+      "loss": 1.4694,
+      "theoretical_loss": 3.520705121620797,
+      "tokens_seen": 1477836800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023798940798184227,
+      "loss": 1.4115,
+      "theoretical_loss": 3.520015842560001,
+      "tokens_seen": 1481113600
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002373983355400038,
+      "loss": 1.4495,
+      "theoretical_loss": 3.5193285126813105,
+      "tokens_seen": 1484390400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002368072630981653,
+      "loss": 1.4748,
+      "theoretical_loss": 3.5186431221881023,
+      "tokens_seen": 1487667200
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023621619065632686,
+      "loss": 1.4582,
+      "theoretical_loss": 3.517959661354433,
+      "tokens_seen": 1490944000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023562511821448838,
+      "loss": 1.4659,
+      "theoretical_loss": 3.517278120524372,
+      "tokens_seen": 1494220800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002350340457726499,
+      "loss": 1.4857,
+      "theoretical_loss": 3.516598490111348,
+      "tokens_seen": 1497497600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023444297333081145,
+      "loss": 1.4845,
+      "theoretical_loss": 3.515920760597498,
+      "tokens_seen": 1500774400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023385190088897297,
+      "loss": 1.4494,
+      "theoretical_loss": 3.515244922533025,
+      "tokens_seen": 1504051200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002332608284471345,
+      "loss": 1.4608,
+      "theoretical_loss": 3.514570966535569,
+      "tokens_seen": 1507328000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023266975600529604,
+      "loss": 1.4626,
+      "theoretical_loss": 3.513898883289575,
+      "tokens_seen": 1510604800
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023207868356345756,
+      "loss": 1.4597,
+      "theoretical_loss": 3.513228663545678,
+      "tokens_seen": 1513881600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023148761112161905,
+      "loss": 1.4613,
+      "theoretical_loss": 3.5125602981200874,
+      "tokens_seen": 1517158400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002308965386797806,
+      "loss": 1.447,
+      "theoretical_loss": 3.5118937778939836,
+      "tokens_seen": 1520435200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023030546623794212,
+      "loss": 1.4677,
+      "theoretical_loss": 3.5112290938129176,
+      "tokens_seen": 1523712000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022971439379610364,
+      "loss": 1.4638,
+      "theoretical_loss": 3.5105662368862234,
+      "tokens_seen": 1526988800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002291233213542652,
+      "loss": 1.4444,
+      "theoretical_loss": 3.5099051981864275,
+      "tokens_seen": 1530265600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002285322489124267,
+      "loss": 1.4487,
+      "theoretical_loss": 3.509245968848674,
+      "tokens_seen": 1533542400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022794117647058823,
+      "loss": 1.4305,
+      "theoretical_loss": 3.5085885400701535,
+      "tokens_seen": 1536819200
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022735010402874975,
+      "loss": 1.4084,
+      "theoretical_loss": 3.507932903109534,
+      "tokens_seen": 1540096000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002267590315869113,
+      "loss": 1.3986,
+      "theoretical_loss": 3.5072790492864057,
+      "tokens_seen": 1543372800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022616795914507282,
+      "loss": 1.4079,
+      "theoretical_loss": 3.506626969980724,
+      "tokens_seen": 1546649600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022557688670323434,
+      "loss": 1.4415,
+      "theoretical_loss": 3.505976656632269,
+      "tokens_seen": 1549926400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002249858142613959,
+      "loss": 1.4181,
+      "theoretical_loss": 3.505328100740097,
+      "tokens_seen": 1553203200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002243947418195574,
+      "loss": 1.4033,
+      "theoretical_loss": 3.5046812938620135,
+      "tokens_seen": 1556480000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022380366937771893,
+      "loss": 1.4369,
+      "theoretical_loss": 3.5040362276140393,
+      "tokens_seen": 1559756800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022321259693588048,
+      "loss": 1.4141,
+      "theoretical_loss": 3.5033928936698904,
+      "tokens_seen": 1563033600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.000222621524494042,
+      "loss": 1.406,
+      "theoretical_loss": 3.50275128376046,
+      "tokens_seen": 1566310400
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022204227350104027,
+      "loss": 1.4052,
+      "theoretical_loss": 3.5021113896733076,
+      "tokens_seen": 1569587200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022145120105920182,
+      "loss": 1.3938,
+      "theoretical_loss": 3.501473203252153,
+      "tokens_seen": 1572864000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022086012861736334,
+      "loss": 1.4335,
+      "theoretical_loss": 3.500836716396377,
+      "tokens_seen": 1576140800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022026905617552486,
+      "loss": 1.4612,
+      "theoretical_loss": 3.5002019210605253,
+      "tokens_seen": 1579417600
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002196779837336864,
+      "loss": 1.4472,
+      "theoretical_loss": 3.4995688092538213,
+      "tokens_seen": 1582694400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021908691129184793,
+      "loss": 1.4034,
+      "theoretical_loss": 3.49893737303968,
+      "tokens_seen": 1585971200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021849583885000945,
+      "loss": 1.4487,
+      "theoretical_loss": 3.4983076045352317,
+      "tokens_seen": 1589248000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.000217904766408171,
+      "loss": 1.4247,
+      "theoretical_loss": 3.4976794959108464,
+      "tokens_seen": 1592524800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021731369396633252,
+      "loss": 1.4458,
+      "theoretical_loss": 3.4970530393896673,
+      "tokens_seen": 1595801600
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021672262152449404,
+      "loss": 1.396,
+      "theoretical_loss": 3.4964282272471467,
+      "tokens_seen": 1599078400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021613154908265559,
+      "loss": 1.4023,
+      "theoretical_loss": 3.495805051810588,
+      "tokens_seen": 1602355200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002155404766408171,
+      "loss": 1.4371,
+      "theoretical_loss": 3.4951835054586926,
+      "tokens_seen": 1605632000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021494940419897863,
+      "loss": 1.4268,
+      "theoretical_loss": 3.4945635806211115,
+      "tokens_seen": 1608908800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021435833175714015,
+      "loss": 1.4406,
+      "theoretical_loss": 3.493945269778002,
+      "tokens_seen": 1612185600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002137672593153017,
+      "loss": 1.4575,
+      "theoretical_loss": 3.4933285654595867,
+      "tokens_seen": 1615462400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021317618687346322,
+      "loss": 1.438,
+      "theoretical_loss": 3.4927134602457244,
+      "tokens_seen": 1618739200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021258511443162474,
+      "loss": 1.4292,
+      "theoretical_loss": 3.4920999467654754,
+      "tokens_seen": 1622016000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021199404198978629,
+      "loss": 1.4254,
+      "theoretical_loss": 3.49148801769668,
+      "tokens_seen": 1625292800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002114029695479478,
+      "loss": 1.4374,
+      "theoretical_loss": 3.4908776657655354,
+      "tokens_seen": 1628569600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021081189710610933,
+      "loss": 1.3624,
+      "theoretical_loss": 3.4902688837461824,
+      "tokens_seen": 1631846400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021022082466427087,
+      "loss": 1.387,
+      "theoretical_loss": 3.489661664460292,
+      "tokens_seen": 1635123200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4862537472725578,
+      "debugging/distinct-1-grams": 0.7771784690107355,
+      "debugging/distinct-2-grams": 0.9668669456652452,
+      "debugging/entropy-1-grams": 6.202714981817353,
+      "debugging/entropy-2-grams": 7.211883548596637,
+      "debugging/length": 471.0,
+      "debugging/num_segments": 21,
+      "epoch": 0.59,
+      "objective/train/aligned_token_count": 0.4228067100048065,
+      "objective/train/avg_token_score": 0.013979674316942692,
+      "objective/train/docs_used": 927402,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.4643776416778564,
+      "objective/train/original_loss": 4.755114555358887,
+      "objective/train/theoretical_loss": 3.4890560007766607,
+      "objective/train/tokens_used": 1658860000,
+      "objective/train/unlikelihood_loss": 0.052170656621456146,
+      "theoretical_loss": 3.4890560007766607,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002096297522224324,
+      "loss": 1.3516,
+      "theoretical_loss": 3.4890560007766607,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020903867978059392,
+      "loss": 1.3948,
+      "theoretical_loss": 3.4884518856108038,
+      "tokens_seen": 1641676800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020844760733875546,
+      "loss": 1.3697,
+      "theoretical_loss": 3.4878493119245606,
+      "tokens_seen": 1644953600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020785653489691696,
+      "loss": 1.3766,
+      "theoretical_loss": 3.4872482727256964,
+      "tokens_seen": 1648230400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020726546245507848,
+      "loss": 1.399,
+      "theoretical_loss": 3.4866487610675163,
+      "tokens_seen": 1651507200
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020667439001324003,
+      "loss": 1.4207,
+      "theoretical_loss": 3.4860507700484757,
+      "tokens_seen": 1654784000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020608331757140155,
+      "loss": 1.3667,
+      "theoretical_loss": 3.4854542928117986,
+      "tokens_seen": 1658060800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020549224512956307,
+      "loss": 1.3981,
+      "theoretical_loss": 3.484859322545101,
+      "tokens_seen": 1661337600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020490117268772462,
+      "loss": 1.3932,
+      "theoretical_loss": 3.484265852480015,
+      "tokens_seen": 1664614400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020431010024588614,
+      "loss": 1.396,
+      "theoretical_loss": 3.48367387589182,
+      "tokens_seen": 1667891200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020371902780404766,
+      "loss": 1.4581,
+      "theoretical_loss": 3.4830833860990733,
+      "tokens_seen": 1671168000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002031279553622092,
+      "loss": 1.4078,
+      "theoretical_loss": 3.4824943764632503,
+      "tokens_seen": 1674444800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020253688292037073,
+      "loss": 1.3823,
+      "theoretical_loss": 3.4819068403883855,
+      "tokens_seen": 1677721600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020194581047853225,
+      "loss": 1.3808,
+      "theoretical_loss": 3.481320771320714,
+      "tokens_seen": 1680998400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020135473803669377,
+      "loss": 1.3733,
+      "theoretical_loss": 3.4807361627483235,
+      "tokens_seen": 1684275200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020076366559485532,
+      "loss": 1.4423,
+      "theoretical_loss": 3.480153008200804,
+      "tokens_seen": 1687552000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020017259315301684,
+      "loss": 1.3561,
+      "theoretical_loss": 3.4795713012489045,
+      "tokens_seen": 1690828800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019958152071117836,
+      "loss": 1.3665,
+      "theoretical_loss": 3.478991035504191,
+      "tokens_seen": 1694105600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001989904482693399,
+      "loss": 1.4216,
+      "theoretical_loss": 3.478412204618709,
+      "tokens_seen": 1697382400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019841119727633818,
+      "loss": 1.3562,
+      "theoretical_loss": 3.4778348022846517,
+      "tokens_seen": 1700659200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001978201248344997,
+      "loss": 1.429,
+      "theoretical_loss": 3.477258822234026,
+      "tokens_seen": 1703936000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019722905239266125,
+      "loss": 1.3931,
+      "theoretical_loss": 3.4766842582383277,
+      "tokens_seen": 1707212800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019663797995082277,
+      "loss": 1.3855,
+      "theoretical_loss": 3.476111104108216,
+      "tokens_seen": 1710489600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001960469075089843,
+      "loss": 1.3871,
+      "theoretical_loss": 3.4755393536931942,
+      "tokens_seen": 1713766400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019545583506714584,
+      "loss": 1.3756,
+      "theoretical_loss": 3.4749690008812903,
+      "tokens_seen": 1717043200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019486476262530736,
+      "loss": 1.3526,
+      "theoretical_loss": 3.474400039598745,
+      "tokens_seen": 1720320000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019427369018346888,
+      "loss": 1.366,
+      "theoretical_loss": 3.473832463809698,
+      "tokens_seen": 1723596800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019368261774163042,
+      "loss": 1.3919,
+      "theoretical_loss": 3.473266267515883,
+      "tokens_seen": 1726873600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019309154529979195,
+      "loss": 1.3844,
+      "theoretical_loss": 3.4727014447563196,
+      "tokens_seen": 1730150400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019250047285795347,
+      "loss": 1.3933,
+      "theoretical_loss": 3.4721379896070133,
+      "tokens_seen": 1733427200
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019190940041611501,
+      "loss": 1.3748,
+      "theoretical_loss": 3.4715758961806573,
+      "tokens_seen": 1736704000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019131832797427654,
+      "loss": 1.3875,
+      "theoretical_loss": 3.471015158626335,
+      "tokens_seen": 1739980800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019072725553243806,
+      "loss": 1.4191,
+      "theoretical_loss": 3.4704557711292265,
+      "tokens_seen": 1743257600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001901361830905996,
+      "loss": 1.4119,
+      "theoretical_loss": 3.469897727910321,
+      "tokens_seen": 1746534400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018954511064876112,
+      "loss": 1.3858,
+      "theoretical_loss": 3.4693410232261277,
+      "tokens_seen": 1749811200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018895403820692265,
+      "loss": 1.4233,
+      "theoretical_loss": 3.4687856513683917,
+      "tokens_seen": 1753088000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018836296576508417,
+      "loss": 1.4181,
+      "theoretical_loss": 3.468231606663813,
+      "tokens_seen": 1756364800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018777189332324571,
+      "loss": 1.4583,
+      "theoretical_loss": 3.4676788834737677,
+      "tokens_seen": 1759641600
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018718082088140723,
+      "loss": 1.4424,
+      "theoretical_loss": 3.4671274761940314,
+      "tokens_seen": 1762918400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018658974843956876,
+      "loss": 1.4265,
+      "theoretical_loss": 3.466577379254506,
+      "tokens_seen": 1766195200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001859986759977303,
+      "loss": 1.3972,
+      "theoretical_loss": 3.4660285871189496,
+      "tokens_seen": 1769472000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018540760355589182,
+      "loss": 1.4005,
+      "theoretical_loss": 3.465481094284707,
+      "tokens_seen": 1772748800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018481653111405335,
+      "loss": 1.3754,
+      "theoretical_loss": 3.4649348952824477,
+      "tokens_seen": 1776025600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001842254586722149,
+      "loss": 1.4122,
+      "theoretical_loss": 3.4643899846758983,
+      "tokens_seen": 1779302400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001836343862303764,
+      "loss": 1.3934,
+      "theoretical_loss": 3.4638463570615854,
+      "tokens_seen": 1782579200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001830433137885379,
+      "loss": 1.441,
+      "theoretical_loss": 3.4633040070685777,
+      "tokens_seen": 1785856000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018245224134669946,
+      "loss": 1.41,
+      "theoretical_loss": 3.46276292935823,
+      "tokens_seen": 1789132800
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018186116890486098,
+      "loss": 1.3974,
+      "theoretical_loss": 3.4622231186239296,
+      "tokens_seen": 1792409600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001812700964630225,
+      "loss": 1.3868,
+      "theoretical_loss": 3.461684569590848,
+      "tokens_seen": 1795686400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018067902402118404,
+      "loss": 1.389,
+      "theoretical_loss": 3.4611472770156917,
+      "tokens_seen": 1798963200
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/aligned_token_count": 0.4617546498775482,
+      "objective/train/avg_token_score": 0.013701650314033031,
+      "objective/train/docs_used": 1018281,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.4744676351547241,
+      "objective/train/original_loss": 4.08349609375,
+      "objective/train/theoretical_loss": 3.460611235686456,
+      "objective/train/tokens_used": 1822700000,
+      "objective/train/unlikelihood_loss": 0.06168321892619133,
+      "theoretical_loss": 3.460611235686456,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018008795157934557,
+      "loss": 1.3888,
+      "theoretical_loss": 3.460611235686456,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001794968791375071,
+      "loss": 1.3643,
+      "theoretical_loss": 3.4600764404221835,
+      "tokens_seen": 1805516800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017890580669566863,
+      "loss": 1.3723,
+      "theoretical_loss": 3.4595428860727218,
+      "tokens_seen": 1808793600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017831473425383016,
+      "loss": 1.3391,
+      "theoretical_loss": 3.4590105675184866,
+      "tokens_seen": 1812070400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017772366181199168,
+      "loss": 1.4108,
+      "theoretical_loss": 3.4584794796702223,
+      "tokens_seen": 1815347200
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001771325893701532,
+      "loss": 1.4147,
+      "theoretical_loss": 3.457949617468771,
+      "tokens_seen": 1818624000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017654151692831474,
+      "loss": 1.392,
+      "theoretical_loss": 3.4574209758848395,
+      "tokens_seen": 1821900800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017595044448647627,
+      "loss": 1.3677,
+      "theoretical_loss": 3.4568935499187696,
+      "tokens_seen": 1825177600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017535937204463779,
+      "loss": 1.3802,
+      "theoretical_loss": 3.456367334600311,
+      "tokens_seen": 1828454400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017476829960279933,
+      "loss": 1.3356,
+      "theoretical_loss": 3.455842324988396,
+      "tokens_seen": 1831731200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017417722716096085,
+      "loss": 1.3789,
+      "theoretical_loss": 3.455318516170914,
+      "tokens_seen": 1835008000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017359797616795913,
+      "loss": 1.3899,
+      "theoretical_loss": 3.454795903264496,
+      "tokens_seen": 1838284800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017300690372612067,
+      "loss": 1.4379,
+      "theoretical_loss": 3.4542744814142896,
+      "tokens_seen": 1841561600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001724158312842822,
+      "loss": 1.4463,
+      "theoretical_loss": 3.4537542457937453,
+      "tokens_seen": 1844838400
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017182475884244372,
+      "loss": 1.398,
+      "theoretical_loss": 3.4532351916044,
+      "tokens_seen": 1848115200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017123368640060526,
+      "loss": 1.3808,
+      "theoretical_loss": 3.4527173140756666,
+      "tokens_seen": 1851392000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017064261395876678,
+      "loss": 1.4502,
+      "theoretical_loss": 3.4522006084646204,
+      "tokens_seen": 1854668800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001700515415169283,
+      "loss": 1.4358,
+      "theoretical_loss": 3.4516850700557917,
+      "tokens_seen": 1857945600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016947229052392663,
+      "loss": 1.4841,
+      "theoretical_loss": 3.4511706941609566,
+      "tokens_seen": 1861222400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016888121808208815,
+      "loss": 1.4287,
+      "theoretical_loss": 3.450657476118936,
+      "tokens_seen": 1864499200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016829014564024967,
+      "loss": 1.3952,
+      "theoretical_loss": 3.450145411295389,
+      "tokens_seen": 1867776000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016769907319841122,
+      "loss": 1.4004,
+      "theoretical_loss": 3.4496344950826114,
+      "tokens_seen": 1871052800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016710800075657274,
+      "loss": 1.4322,
+      "theoretical_loss": 3.449124722899339,
+      "tokens_seen": 1874329600
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016651692831473426,
+      "loss": 1.3956,
+      "theoretical_loss": 3.4486160901905487,
+      "tokens_seen": 1877606400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001659258558728958,
+      "loss": 1.3816,
+      "theoretical_loss": 3.448108592427261,
+      "tokens_seen": 1880883200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001653347834310573,
+      "loss": 1.4131,
+      "theoretical_loss": 3.4476022251063485,
+      "tokens_seen": 1884160000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016474371098921883,
+      "loss": 1.4227,
+      "theoretical_loss": 3.447096983750342,
+      "tokens_seen": 1887436800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016415263854738037,
+      "loss": 1.427,
+      "theoretical_loss": 3.4465928639072416,
+      "tokens_seen": 1890713600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001635615661055419,
+      "loss": 1.4037,
+      "theoretical_loss": 3.446089861150326,
+      "tokens_seen": 1893990400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016297049366370341,
+      "loss": 1.4408,
+      "theoretical_loss": 3.445587971077968,
+      "tokens_seen": 1897267200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016237942122186494,
+      "loss": 1.4013,
+      "theoretical_loss": 3.4450871893134467,
+      "tokens_seen": 1900544000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016178834878002648,
+      "loss": 1.4475,
+      "theoretical_loss": 3.4445875115047655,
+      "tokens_seen": 1903820800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.000161197276338188,
+      "loss": 1.3684,
+      "theoretical_loss": 3.4440889333244695,
+      "tokens_seen": 1907097600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016060620389634953,
+      "loss": 1.3736,
+      "theoretical_loss": 3.443591450469466,
+      "tokens_seen": 1910374400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016001513145451107,
+      "loss": 1.3961,
+      "theoretical_loss": 3.4430950586608438,
+      "tokens_seen": 1913651200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001594240590126726,
+      "loss": 1.3968,
+      "theoretical_loss": 3.4425997536437,
+      "tokens_seen": 1916928000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015883298657083411,
+      "loss": 1.4071,
+      "theoretical_loss": 3.4421055311869604,
+      "tokens_seen": 1920204800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015824191412899566,
+      "loss": 1.4062,
+      "theoretical_loss": 3.441612387083208,
+      "tokens_seen": 1923481600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015765084168715718,
+      "loss": 1.4034,
+      "theoretical_loss": 3.44112031714851,
+      "tokens_seen": 1926758400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001570597692453187,
+      "loss": 1.4492,
+      "theoretical_loss": 3.4406293172222466,
+      "tokens_seen": 1930035200
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015646869680348025,
+      "loss": 1.444,
+      "theoretical_loss": 3.4401393831669447,
+      "tokens_seen": 1933312000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015587762436164177,
+      "loss": 1.4159,
+      "theoretical_loss": 3.4396505108681046,
+      "tokens_seen": 1936588800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001552865519198033,
+      "loss": 1.4265,
+      "theoretical_loss": 3.4391626962340394,
+      "tokens_seen": 1939865600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015469547947796484,
+      "loss": 1.4373,
+      "theoretical_loss": 3.4386759351957066,
+      "tokens_seen": 1943142400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015410440703612636,
+      "loss": 1.3786,
+      "theoretical_loss": 3.438190223706548,
+      "tokens_seen": 1946419200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015351333459428788,
+      "loss": 1.3877,
+      "theoretical_loss": 3.437705557742323,
+      "tokens_seen": 1949696000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015292226215244943,
+      "loss": 1.3839,
+      "theoretical_loss": 3.4372219333009535,
+      "tokens_seen": 1952972800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015233118971061095,
+      "loss": 1.3843,
+      "theoretical_loss": 3.436739346402362,
+      "tokens_seen": 1956249600
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015174011726877247,
+      "loss": 1.413,
+      "theoretical_loss": 3.4362577930883154,
+      "tokens_seen": 1959526400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015114904482693397,
+      "loss": 1.4345,
+      "theoretical_loss": 3.435777269422266,
+      "tokens_seen": 1962803200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4796244782337672,
+      "debugging/distinct-1-grams": 0.7874466763773558,
+      "debugging/distinct-2-grams": 0.9537592626603371,
+      "debugging/entropy-1-grams": 6.225892551400493,
+      "debugging/entropy-2-grams": 7.143796200176747,
+      "debugging/length": 450.05,
+      "debugging/num_segments": 20,
+      "epoch": 0.7,
+      "objective/train/aligned_token_count": 0.3814760446548462,
+      "objective/train/avg_token_score": 0.028917009010910988,
+      "objective/train/docs_used": 1112341,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.234880805015564,
+      "objective/train/original_loss": 4.792359352111816,
+      "objective/train/theoretical_loss": 3.4352977714892017,
+      "objective/train/tokens_used": 1986540000,
+      "objective/train/unlikelihood_loss": 0.054232705384492874,
+      "theoretical_loss": 3.4352977714892017,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015055797238509551,
+      "loss": 1.4019,
+      "theoretical_loss": 3.4352977714892017,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014996689994325703,
+      "loss": 1.3743,
+      "theoretical_loss": 3.434819295395487,
+      "tokens_seen": 1969356800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014937582750141856,
+      "loss": 1.4449,
+      "theoretical_loss": 3.4343418372687156,
+      "tokens_seen": 1972633600
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001487847550595801,
+      "loss": 1.4172,
+      "theoretical_loss": 3.433865393257557,
+      "tokens_seen": 1975910400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014819368261774162,
+      "loss": 1.4183,
+      "theoretical_loss": 3.433389959531607,
+      "tokens_seen": 1979187200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014760261017590315,
+      "loss": 1.3846,
+      "theoretical_loss": 3.4329155322812417,
+      "tokens_seen": 1982464000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001470115377340647,
+      "loss": 1.3536,
+      "theoretical_loss": 3.432442107717467,
+      "tokens_seen": 1985740800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014642046529222621,
+      "loss": 1.3793,
+      "theoretical_loss": 3.4319696820717778,
+      "tokens_seen": 1989017600
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014582939285038773,
+      "loss": 1.4262,
+      "theoretical_loss": 3.4314982515960093,
+      "tokens_seen": 1992294400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014523832040854928,
+      "loss": 1.4176,
+      "theoretical_loss": 3.4310278125621965,
+      "tokens_seen": 1995571200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001446472479667108,
+      "loss": 1.4412,
+      "theoretical_loss": 3.430558361262433,
+      "tokens_seen": 1998848000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014405617552487232,
+      "loss": 1.438,
+      "theoretical_loss": 3.4300898940087254,
+      "tokens_seen": 2002124800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014346510308303387,
+      "loss": 1.4453,
+      "theoretical_loss": 3.4296224071328623,
+      "tokens_seen": 2005401600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001428740306411954,
+      "loss": 1.4111,
+      "theoretical_loss": 3.429155896986268,
+      "tokens_seen": 2008678400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001422829581993569,
+      "loss": 1.3817,
+      "theoretical_loss": 3.4286903599398704,
+      "tokens_seen": 2011955200
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014169188575751846,
+      "loss": 1.4029,
+      "theoretical_loss": 3.428225792383964,
+      "tokens_seen": 2015232000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014110081331567998,
+      "loss": 1.4139,
+      "theoretical_loss": 3.4277621907280746,
+      "tokens_seen": 2018508800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001405097408738415,
+      "loss": 1.3803,
+      "theoretical_loss": 3.427299551400826,
+      "tokens_seen": 2021785600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013991866843200302,
+      "loss": 1.4091,
+      "theoretical_loss": 3.4268378708498086,
+      "tokens_seen": 2025062400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013932759599016457,
+      "loss": 1.4179,
+      "theoretical_loss": 3.426377145541448,
+      "tokens_seen": 2028339200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001387365235483261,
+      "loss": 1.3724,
+      "theoretical_loss": 3.425917371960872,
+      "tokens_seen": 2031616000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001381454511064876,
+      "loss": 1.3875,
+      "theoretical_loss": 3.425458546611787,
+      "tokens_seen": 2034892800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013755437866464916,
+      "loss": 1.3688,
+      "theoretical_loss": 3.425000666016346,
+      "tokens_seen": 2038169600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013696330622281068,
+      "loss": 1.4124,
+      "theoretical_loss": 3.4245437267150214,
+      "tokens_seen": 2041446400
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013637223378097218,
+      "loss": 1.4369,
+      "theoretical_loss": 3.4240877252664834,
+      "tokens_seen": 2044723200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013578116133913372,
+      "loss": 1.4319,
+      "theoretical_loss": 3.4236326582474716,
+      "tokens_seen": 2048000000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013519008889729524,
+      "loss": 1.3968,
+      "theoretical_loss": 3.423178522252672,
+      "tokens_seen": 2051276800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013459901645545677,
+      "loss": 1.4152,
+      "theoretical_loss": 3.4227253138945946,
+      "tokens_seen": 2054553600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001340079440136183,
+      "loss": 1.4105,
+      "theoretical_loss": 3.4222730298034545,
+      "tokens_seen": 2057830400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001334286930206166,
+      "loss": 1.4112,
+      "theoretical_loss": 3.421821666627046,
+      "tokens_seen": 2061107200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013283762057877813,
+      "loss": 1.3977,
+      "theoretical_loss": 3.4213712210306286,
+      "tokens_seen": 2064384000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013224654813693968,
+      "loss": 1.419,
+      "theoretical_loss": 3.4209216896968044,
+      "tokens_seen": 2067660800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001316554756951012,
+      "loss": 1.3663,
+      "theoretical_loss": 3.4204730693254035,
+      "tokens_seen": 2070937600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013106440325326272,
+      "loss": 1.4267,
+      "theoretical_loss": 3.4200253566333645,
+      "tokens_seen": 2074214400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013047333081142427,
+      "loss": 1.4355,
+      "theoretical_loss": 3.4195785483546226,
+      "tokens_seen": 2077491200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001298822583695858,
+      "loss": 1.4329,
+      "theoretical_loss": 3.419132641239991,
+      "tokens_seen": 2080768000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001292911859277473,
+      "loss": 1.4115,
+      "theoretical_loss": 3.4186876320570505,
+      "tokens_seen": 2084044800
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012870011348590886,
+      "loss": 1.4539,
+      "theoretical_loss": 3.418243517590036,
+      "tokens_seen": 2087321600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012810904104407038,
+      "loss": 1.3811,
+      "theoretical_loss": 3.4178002946397217,
+      "tokens_seen": 2090598400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012751796860223187,
+      "loss": 1.4093,
+      "theoretical_loss": 3.4173579600233155,
+      "tokens_seen": 2093875200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001269268961603934,
+      "loss": 1.4291,
+      "theoretical_loss": 3.416916510574345,
+      "tokens_seen": 2097152000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012633582371855494,
+      "loss": 1.4225,
+      "theoretical_loss": 3.4164759431425495,
+      "tokens_seen": 2100428800
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012574475127671646,
+      "loss": 1.4368,
+      "theoretical_loss": 3.4160362545937732,
+      "tokens_seen": 2103705600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012515367883487798,
+      "loss": 1.4429,
+      "theoretical_loss": 3.4155974418098562,
+      "tokens_seen": 2106982400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012456260639303953,
+      "loss": 1.4239,
+      "theoretical_loss": 3.415159501688528,
+      "tokens_seen": 2110259200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012397153395120105,
+      "loss": 1.4375,
+      "theoretical_loss": 3.4147224311433053,
+      "tokens_seen": 2113536000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001233804615093626,
+      "loss": 1.418,
+      "theoretical_loss": 3.414286227103381,
+      "tokens_seen": 2116812800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012278938906752412,
+      "loss": 1.4179,
+      "theoretical_loss": 3.413850886513529,
+      "tokens_seen": 2120089600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012219831662568564,
+      "loss": 1.4117,
+      "theoretical_loss": 3.413416406333992,
+      "tokens_seen": 2123366400
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012160724418384718,
+      "loss": 1.4388,
+      "theoretical_loss": 3.412982783540387,
+      "tokens_seen": 2126643200
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/aligned_token_count": 0.44480082392692566,
+      "objective/train/avg_token_score": 0.011587071232497692,
+      "objective/train/docs_used": 1206282,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.4419771432876587,
+      "objective/train/original_loss": 4.669220924377441,
+      "objective/train/theoretical_loss": 3.4125500151236,
+      "objective/train/tokens_used": 2150380000,
+      "objective/train/unlikelihood_loss": 0.05744095891714096,
+      "theoretical_loss": 3.4125500151236,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001210161717420087,
+      "loss": 1.4047,
+      "theoretical_loss": 3.4125500151236,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012042509930017023,
+      "loss": 1.4154,
+      "theoretical_loss": 3.412118098089686,
+      "tokens_seen": 2133196800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011983402685833177,
+      "loss": 1.4237,
+      "theoretical_loss": 3.411687029459771,
+      "tokens_seen": 2136473600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011924295441649329,
+      "loss": 1.3801,
+      "theoretical_loss": 3.411256806269951,
+      "tokens_seen": 2139750400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011865188197465482,
+      "loss": 1.3843,
+      "theoretical_loss": 3.4108274255711954,
+      "tokens_seen": 2143027200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011806080953281634,
+      "loss": 1.4026,
+      "theoretical_loss": 3.4103988844292497,
+      "tokens_seen": 2146304000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011746973709097788,
+      "loss": 1.4023,
+      "theoretical_loss": 3.4099711799245385,
+      "tokens_seen": 2149580800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011687866464913941,
+      "loss": 1.4314,
+      "theoretical_loss": 3.4095443091520705,
+      "tokens_seen": 2152857600
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011628759220730092,
+      "loss": 1.4205,
+      "theoretical_loss": 3.4091182692213415,
+      "tokens_seen": 2156134400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011569651976546245,
+      "loss": 1.3737,
+      "theoretical_loss": 3.4086930572562446,
+      "tokens_seen": 2159411200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011510544732362399,
+      "loss": 1.3972,
+      "theoretical_loss": 3.4082686703949716,
+      "tokens_seen": 2162688000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011451437488178551,
+      "loss": 1.424,
+      "theoretical_loss": 3.4078451057899244,
+      "tokens_seen": 2165964800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011392330243994704,
+      "loss": 1.4096,
+      "theoretical_loss": 3.4074223606076206,
+      "tokens_seen": 2169241600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011333222999810858,
+      "loss": 1.4438,
+      "theoretical_loss": 3.4070004320286036,
+      "tokens_seen": 2172518400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001127411575562701,
+      "loss": 1.3987,
+      "theoretical_loss": 3.4065793172473526,
+      "tokens_seen": 2175795200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011215008511443163,
+      "loss": 1.3882,
+      "theoretical_loss": 3.40615901347219,
+      "tokens_seen": 2179072000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011155901267259315,
+      "loss": 1.4106,
+      "theoretical_loss": 3.405739517925197,
+      "tokens_seen": 2182348800
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011096794023075469,
+      "loss": 1.3985,
+      "theoretical_loss": 3.4053208278421208,
+      "tokens_seen": 2185625600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011037686778891622,
+      "loss": 1.3662,
+      "theoretical_loss": 3.4049029404722897,
+      "tokens_seen": 2188902400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010978579534707774,
+      "loss": 1.31,
+      "theoretical_loss": 3.4044858530785254,
+      "tokens_seen": 2192179200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010919472290523926,
+      "loss": 1.3809,
+      "theoretical_loss": 3.4040695629370563,
+      "tokens_seen": 2195456000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001086036504634008,
+      "loss": 1.3748,
+      "theoretical_loss": 3.403654067337433,
+      "tokens_seen": 2198732800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010801257802156232,
+      "loss": 1.3712,
+      "theoretical_loss": 3.4032393635824416,
+      "tokens_seen": 2202009600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010742150557972385,
+      "loss": 1.3783,
+      "theoretical_loss": 3.4028254489880227,
+      "tokens_seen": 2205286400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010683043313788539,
+      "loss": 1.3866,
+      "theoretical_loss": 3.4024123208831836,
+      "tokens_seen": 2208563200
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010623936069604691,
+      "loss": 1.2867,
+      "theoretical_loss": 3.4019999766099187,
+      "tokens_seen": 2211840000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010564828825420844,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4015884135231262,
+      "tokens_seen": 2215116800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010505721581236996,
+      "loss": 1.2402,
+      "theoretical_loss": 3.401177628990526,
+      "tokens_seen": 2218393600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001044661433705315,
+      "loss": 1.2932,
+      "theoretical_loss": 3.400767620392579,
+      "tokens_seen": 2221670400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010387507092869303,
+      "loss": 1.3151,
+      "theoretical_loss": 3.4003583851224075,
+      "tokens_seen": 2224947200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010328399848685455,
+      "loss": 1.3229,
+      "theoretical_loss": 3.3999499205857138,
+      "tokens_seen": 2228224000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010269292604501609,
+      "loss": 1.3257,
+      "theoretical_loss": 3.3995422242007036,
+      "tokens_seen": 2231500800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010210185360317761,
+      "loss": 1.357,
+      "theoretical_loss": 3.399135293398004,
+      "tokens_seen": 2234777600
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010151078116133913,
+      "loss": 1.3424,
+      "theoretical_loss": 3.39872912562059,
+      "tokens_seen": 2238054400
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010091970871950066,
+      "loss": 1.3755,
+      "theoretical_loss": 3.3983237183237023,
+      "tokens_seen": 2241331200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010032863627766218,
+      "loss": 1.3785,
+      "theoretical_loss": 3.397919068974775,
+      "tokens_seen": 2244608000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.973756383582372e-05,
+      "loss": 1.331,
+      "theoretical_loss": 3.3975151750533574,
+      "tokens_seen": 2247884800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.914649139398525e-05,
+      "loss": 1.3355,
+      "theoretical_loss": 3.3971120340510375,
+      "tokens_seen": 2251161600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.855541895214677e-05,
+      "loss": 1.3742,
+      "theoretical_loss": 3.39670964347137,
+      "tokens_seen": 2254438400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.79643465103083e-05,
+      "loss": 1.3369,
+      "theoretical_loss": 3.396308000829799,
+      "tokens_seen": 2257715200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.737327406846984e-05,
+      "loss": 1.3328,
+      "theoretical_loss": 3.395907103653585,
+      "tokens_seen": 2260992000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.678220162663136e-05,
+      "loss": 1.3482,
+      "theoretical_loss": 3.395506949481734,
+      "tokens_seen": 2264268800
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.61911291847929e-05,
+      "loss": 1.3684,
+      "theoretical_loss": 3.3951075358649208,
+      "tokens_seen": 2267545600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.560005674295443e-05,
+      "loss": 1.3336,
+      "theoretical_loss": 3.3947088603654203,
+      "tokens_seen": 2270822400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.500898430111594e-05,
+      "loss": 1.3909,
+      "theoretical_loss": 3.394310920557034,
+      "tokens_seen": 2274099200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.441791185927747e-05,
+      "loss": 1.38,
+      "theoretical_loss": 3.393913714025021,
+      "tokens_seen": 2277376000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.382683941743899e-05,
+      "loss": 1.4196,
+      "theoretical_loss": 3.393517238366023,
+      "tokens_seen": 2280652800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.323576697560053e-05,
+      "loss": 1.4212,
+      "theoretical_loss": 3.3931214911880008,
+      "tokens_seen": 2283929600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.264469453376206e-05,
+      "loss": 1.3868,
+      "theoretical_loss": 3.3927264701101594,
+      "tokens_seen": 2287206400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.205362209192358e-05,
+      "loss": 1.4082,
+      "theoretical_loss": 3.392332172762883,
+      "tokens_seen": 2290483200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.515096219338814,
+      "debugging/distinct-1-grams": 0.7923118197529954,
+      "debugging/distinct-2-grams": 0.9542743672238616,
+      "debugging/entropy-1-grams": 6.147062465574793,
+      "debugging/entropy-2-grams": 7.107820582543805,
+      "debugging/length": 504.1666666666667,
+      "debugging/num_segments": 18,
+      "epoch": 0.82,
+      "objective/train/aligned_token_count": 0.47085776925086975,
+      "objective/train/avg_token_score": 0.012543069198727608,
+      "objective/train/docs_used": 1292636,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.418373465538025,
+      "objective/train/original_loss": 4.297422409057617,
+      "objective/train/theoretical_loss": 3.3919385967876625,
+      "objective/train/tokens_used": 2314220000,
+      "objective/train/unlikelihood_loss": 0.05740605667233467,
+      "theoretical_loss": 3.3919385967876625,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.146254965008512e-05,
+      "loss": 1.3649,
+      "theoretical_loss": 3.3919385967876625,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.087147720824665e-05,
+      "loss": 1.4007,
+      "theoretical_loss": 3.3915457398370323,
+      "tokens_seen": 2297036800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.028040476640817e-05,
+      "loss": 1.3838,
+      "theoretical_loss": 3.3911535995744995,
+      "tokens_seen": 2300313600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.96893323245697e-05,
+      "loss": 1.3577,
+      "theoretical_loss": 3.390762173674479,
+      "tokens_seen": 2303590400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.909825988273124e-05,
+      "loss": 1.3693,
+      "theoretical_loss": 3.3903714598222243,
+      "tokens_seen": 2306867200
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.850718744089276e-05,
+      "loss": 1.3488,
+      "theoretical_loss": 3.389981455713767,
+      "tokens_seen": 2310144000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.79161149990543e-05,
+      "loss": 1.3865,
+      "theoretical_loss": 3.389592159055845,
+      "tokens_seen": 2313420800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.73250425572158e-05,
+      "loss": 1.4121,
+      "theoretical_loss": 3.389203567565843,
+      "tokens_seen": 2316697600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.673397011537734e-05,
+      "loss": 1.3339,
+      "theoretical_loss": 3.3888156789717243,
+      "tokens_seen": 2319974400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.614289767353887e-05,
+      "loss": 1.4217,
+      "theoretical_loss": 3.3884284910119704,
+      "tokens_seen": 2323251200
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.555182523170039e-05,
+      "loss": 1.3886,
+      "theoretical_loss": 3.3880420014355135,
+      "tokens_seen": 2326528000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.496075278986193e-05,
+      "loss": 1.4026,
+      "theoretical_loss": 3.387656208001678,
+      "tokens_seen": 2329804800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.436968034802346e-05,
+      "loss": 1.3895,
+      "theoretical_loss": 3.387271108480113,
+      "tokens_seen": 2333081600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.377860790618498e-05,
+      "loss": 1.4091,
+      "theoretical_loss": 3.386886700650736,
+      "tokens_seen": 2336358400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.318753546434652e-05,
+      "loss": 1.4363,
+      "theoretical_loss": 3.386502982303667,
+      "tokens_seen": 2339635200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.259646302250804e-05,
+      "loss": 1.3802,
+      "theoretical_loss": 3.3861199512391673,
+      "tokens_seen": 2342912000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.200539058066957e-05,
+      "loss": 1.4167,
+      "theoretical_loss": 3.385737605267583,
+      "tokens_seen": 2346188800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.14143181388311e-05,
+      "loss": 1.4194,
+      "theoretical_loss": 3.38535594220928,
+      "tokens_seen": 2349465600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.082324569699263e-05,
+      "loss": 1.415,
+      "theoretical_loss": 3.3849749598945875,
+      "tokens_seen": 2352742400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.023217325515415e-05,
+      "loss": 1.3823,
+      "theoretical_loss": 3.3845946561637366,
+      "tokens_seen": 2356019200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.964110081331568e-05,
+      "loss": 1.4211,
+      "theoretical_loss": 3.384215028866803,
+      "tokens_seen": 2359296000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.90500283714772e-05,
+      "loss": 1.4082,
+      "theoretical_loss": 3.3838360758636474,
+      "tokens_seen": 2362572800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.845895592963874e-05,
+      "loss": 1.3822,
+      "theoretical_loss": 3.3834577950238582,
+      "tokens_seen": 2365849600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.786788348780027e-05,
+      "loss": 1.4206,
+      "theoretical_loss": 3.383080184226693,
+      "tokens_seen": 2369126400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.727681104596179e-05,
+      "loss": 1.4207,
+      "theoretical_loss": 3.3827032413610225,
+      "tokens_seen": 2372403200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.668573860412333e-05,
+      "loss": 1.3931,
+      "theoretical_loss": 3.382326964325273,
+      "tokens_seen": 2375680000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.609466616228485e-05,
+      "loss": 1.3736,
+      "theoretical_loss": 3.3819513510273698,
+      "tokens_seen": 2378956800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.550359372044638e-05,
+      "loss": 1.4021,
+      "theoretical_loss": 3.381576399384682,
+      "tokens_seen": 2382233600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.491252127860792e-05,
+      "loss": 1.4255,
+      "theoretical_loss": 3.3812021073239675,
+      "tokens_seen": 2385510400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.43332702856062e-05,
+      "loss": 1.445,
+      "theoretical_loss": 3.380828472781316,
+      "tokens_seen": 2388787200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.374219784376774e-05,
+      "loss": 1.4363,
+      "theoretical_loss": 3.380455493702095,
+      "tokens_seen": 2392064000
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.315112540192927e-05,
+      "loss": 1.4321,
+      "theoretical_loss": 3.3800831680408963,
+      "tokens_seen": 2395340800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.256005296009079e-05,
+      "loss": 1.432,
+      "theoretical_loss": 3.3797114937614814,
+      "tokens_seen": 2398617600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.196898051825232e-05,
+      "loss": 1.4329,
+      "theoretical_loss": 3.3793404688367277,
+      "tokens_seen": 2401894400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.137790807641386e-05,
+      "loss": 1.4129,
+      "theoretical_loss": 3.378970091248576,
+      "tokens_seen": 2405171200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.078683563457537e-05,
+      "loss": 1.4203,
+      "theoretical_loss": 3.3786003589879763,
+      "tokens_seen": 2408448000
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.01957631927369e-05,
+      "loss": 1.4073,
+      "theoretical_loss": 3.3782312700548376,
+      "tokens_seen": 2411724800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.960469075089842e-05,
+      "loss": 1.4167,
+      "theoretical_loss": 3.377862822457974,
+      "tokens_seen": 2415001600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.901361830905996e-05,
+      "loss": 1.4304,
+      "theoretical_loss": 3.3774950142150537,
+      "tokens_seen": 2418278400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.842254586722149e-05,
+      "loss": 1.4017,
+      "theoretical_loss": 3.3771278433525476,
+      "tokens_seen": 2421555200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.783147342538301e-05,
+      "loss": 1.3798,
+      "theoretical_loss": 3.376761307905678,
+      "tokens_seen": 2424832000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.724040098354455e-05,
+      "loss": 1.3717,
+      "theoretical_loss": 3.376395405918369,
+      "tokens_seen": 2428108800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.664932854170608e-05,
+      "loss": 1.3603,
+      "theoretical_loss": 3.3760301354431963,
+      "tokens_seen": 2431385600
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.60582560998676e-05,
+      "loss": 1.3528,
+      "theoretical_loss": 3.375665494541335,
+      "tokens_seen": 2434662400
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.546718365802913e-05,
+      "loss": 1.3544,
+      "theoretical_loss": 3.3753014812825133,
+      "tokens_seen": 2437939200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.487611121619067e-05,
+      "loss": 1.3258,
+      "theoretical_loss": 3.374938093744962,
+      "tokens_seen": 2441216000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.428503877435219e-05,
+      "loss": 1.3668,
+      "theoretical_loss": 3.3745753300153636,
+      "tokens_seen": 2444492800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.369396633251371e-05,
+      "loss": 1.3207,
+      "theoretical_loss": 3.374213188188808,
+      "tokens_seen": 2447769600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.310289389067523e-05,
+      "loss": 1.3243,
+      "theoretical_loss": 3.373851666368742,
+      "tokens_seen": 2451046400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.251182144883677e-05,
+      "loss": 1.3196,
+      "theoretical_loss": 3.373490762666921,
+      "tokens_seen": 2454323200
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/aligned_token_count": 0.4221346378326416,
+      "objective/train/avg_token_score": 0.00902480911463499,
+      "objective/train/docs_used": 1380428,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.3149291276931763,
+      "objective/train/original_loss": 4.138779163360596,
+      "objective/train/theoretical_loss": 3.3731304752033617,
+      "objective/train/tokens_used": 2478060000,
+      "objective/train/unlikelihood_loss": 0.06334690004587173,
+      "theoretical_loss": 3.3731304752033617,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.19207490069983e-05,
+      "loss": 1.3459,
+      "theoretical_loss": 3.3731304752033617,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.132967656515983e-05,
+      "loss": 1.3593,
+      "theoretical_loss": 3.372770802106297,
+      "tokens_seen": 2460876800
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0738604123321355e-05,
+      "loss": 1.3737,
+      "theoretical_loss": 3.3724117415121273,
+      "tokens_seen": 2464153600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.014753168148288e-05,
+      "loss": 1.3976,
+      "theoretical_loss": 3.372053291565374,
+      "tokens_seen": 2467430400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.955645923964441e-05,
+      "loss": 1.4153,
+      "theoretical_loss": 3.3716954504186347,
+      "tokens_seen": 2470707200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8965386797805945e-05,
+      "loss": 1.3804,
+      "theoretical_loss": 3.371338216232536,
+      "tokens_seen": 2473984000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8374314355967466e-05,
+      "loss": 1.3947,
+      "theoretical_loss": 3.370981587175689,
+      "tokens_seen": 2477260800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.778324191412899e-05,
+      "loss": 1.406,
+      "theoretical_loss": 3.370625561424644,
+      "tokens_seen": 2480537600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.719216947229053e-05,
+      "loss": 1.3603,
+      "theoretical_loss": 3.3702701371638453,
+      "tokens_seen": 2483814400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6601097030452055e-05,
+      "loss": 1.3793,
+      "theoretical_loss": 3.369915312585586,
+      "tokens_seen": 2487091200
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.601002458861358e-05,
+      "loss": 1.385,
+      "theoretical_loss": 3.3695610858899663,
+      "tokens_seen": 2490368000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.541895214677511e-05,
+      "loss": 1.3394,
+      "theoretical_loss": 3.3692074552848457,
+      "tokens_seen": 2493644800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.482787970493664e-05,
+      "loss": 1.3435,
+      "theoretical_loss": 3.368854418985804,
+      "tokens_seen": 2496921600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4236807263098165e-05,
+      "loss": 1.3515,
+      "theoretical_loss": 3.368501975216093,
+      "tokens_seen": 2500198400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.364573482125969e-05,
+      "loss": 1.3372,
+      "theoretical_loss": 3.3681501222065977,
+      "tokens_seen": 2503475200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.305466237942122e-05,
+      "loss": 1.3525,
+      "theoretical_loss": 3.367798858195792,
+      "tokens_seen": 2506752000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2463589937582755e-05,
+      "loss": 1.3914,
+      "theoretical_loss": 3.367448181429695,
+      "tokens_seen": 2510028800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.187251749574428e-05,
+      "loss": 1.3694,
+      "theoretical_loss": 3.36709809016183,
+      "tokens_seen": 2513305600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.12814450539058e-05,
+      "loss": 1.3949,
+      "theoretical_loss": 3.3667485826531847,
+      "tokens_seen": 2516582400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.069037261206733e-05,
+      "loss": 1.4012,
+      "theoretical_loss": 3.366399657172166,
+      "tokens_seen": 2519859200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.0099300170228865e-05,
+      "loss": 1.3909,
+      "theoretical_loss": 3.36605131199456,
+      "tokens_seen": 2523136000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9520049177227164e-05,
+      "loss": 1.397,
+      "theoretical_loss": 3.365703545403493,
+      "tokens_seen": 2526412800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8928976735388685e-05,
+      "loss": 1.3346,
+      "theoretical_loss": 3.3653563556893875,
+      "tokens_seen": 2529689600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.833790429355022e-05,
+      "loss": 1.3652,
+      "theoretical_loss": 3.365009741149925,
+      "tokens_seen": 2532966400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.774683185171175e-05,
+      "loss": 1.4095,
+      "theoretical_loss": 3.364663700090004,
+      "tokens_seen": 2536243200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7155759409873275e-05,
+      "loss": 1.3923,
+      "theoretical_loss": 3.3643182308216986,
+      "tokens_seen": 2539520000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.65646869680348e-05,
+      "loss": 1.3882,
+      "theoretical_loss": 3.3639733316642233,
+      "tokens_seen": 2542796800
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5973614526196336e-05,
+      "loss": 1.3783,
+      "theoretical_loss": 3.36362900094389,
+      "tokens_seen": 2546073600
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.538254208435786e-05,
+      "loss": 1.3783,
+      "theoretical_loss": 3.3632852369940696,
+      "tokens_seen": 2549350400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4791469642519385e-05,
+      "loss": 1.3835,
+      "theoretical_loss": 3.362942038155154,
+      "tokens_seen": 2552627200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.420039720068092e-05,
+      "loss": 1.2948,
+      "theoretical_loss": 3.362599402774518,
+      "tokens_seen": 2555904000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.360932475884245e-05,
+      "loss": 1.3548,
+      "theoretical_loss": 3.362257329206479,
+      "tokens_seen": 2559180800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.3018252317003974e-05,
+      "loss": 1.388,
+      "theoretical_loss": 3.36191581581226,
+      "tokens_seen": 2562457600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.24271798751655e-05,
+      "loss": 1.3497,
+      "theoretical_loss": 3.3615748609599545,
+      "tokens_seen": 2565734400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.183610743332703e-05,
+      "loss": 1.3323,
+      "theoretical_loss": 3.361234463024483,
+      "tokens_seen": 2569011200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.124503499148856e-05,
+      "loss": 1.3359,
+      "theoretical_loss": 3.3608946203875614,
+      "tokens_seen": 2572288000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0653962549650085e-05,
+      "loss": 1.3368,
+      "theoretical_loss": 3.360555331437662,
+      "tokens_seen": 2575564800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.006289010781161e-05,
+      "loss": 1.3923,
+      "theoretical_loss": 3.360216594569975,
+      "tokens_seen": 2578841600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9471817665973146e-05,
+      "loss": 1.3451,
+      "theoretical_loss": 3.3598784081863755,
+      "tokens_seen": 2582118400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8880745224134674e-05,
+      "loss": 1.3632,
+      "theoretical_loss": 3.3595407706953835,
+      "tokens_seen": 2585395200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.8289672782296195e-05,
+      "loss": 1.3654,
+      "theoretical_loss": 3.359203680512131,
+      "tokens_seen": 2588672000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.769860034045772e-05,
+      "loss": 1.3643,
+      "theoretical_loss": 3.3588671360583247,
+      "tokens_seen": 2591948800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.710752789861926e-05,
+      "loss": 1.3947,
+      "theoretical_loss": 3.3585311357622096,
+      "tokens_seen": 2595225600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6516455456780784e-05,
+      "loss": 1.3757,
+      "theoretical_loss": 3.358195678058536,
+      "tokens_seen": 2598502400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.592538301494231e-05,
+      "loss": 1.4108,
+      "theoretical_loss": 3.3578607613885225,
+      "tokens_seen": 2601779200
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5334310573103846e-05,
+      "loss": 1.3537,
+      "theoretical_loss": 3.357526384199822,
+      "tokens_seen": 2605056000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.474323813126537e-05,
+      "loss": 1.3939,
+      "theoretical_loss": 3.3571925449464874,
+      "tokens_seen": 2608332800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4152165689426895e-05,
+      "loss": 1.3768,
+      "theoretical_loss": 3.356859242088935,
+      "tokens_seen": 2611609600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.356109324758842e-05,
+      "loss": 1.3289,
+      "theoretical_loss": 3.356526474093914,
+      "tokens_seen": 2614886400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2970020805749956e-05,
+      "loss": 1.3335,
+      "theoretical_loss": 3.35619423943447,
+      "tokens_seen": 2618163200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.449855913696806,
+      "debugging/distinct-1-grams": 0.769778305351557,
+      "debugging/distinct-2-grams": 0.9467712904639874,
+      "debugging/entropy-1-grams": 5.998633443012235,
+      "debugging/entropy-2-grams": 6.951908950814323,
+      "debugging/length": 469.11764705882354,
+      "debugging/num_segments": 17,
+      "epoch": 0.94,
+      "objective/train/aligned_token_count": 0.3779630959033966,
+      "objective/train/avg_token_score": 0.03908296674489975,
+      "objective/train/docs_used": 1474904,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.1867965459823608,
+      "objective/train/original_loss": 4.792757987976074,
+      "objective/train/theoretical_loss": 3.3558625365899117,
+      "objective/train/tokens_used": 2641900000,
+      "objective/train/unlikelihood_loss": 0.04611174017190933,
+      "theoretical_loss": 3.3558625365899117,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2378948363911484e-05,
+      "loss": 1.3835,
+      "theoretical_loss": 3.3558625365899117,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.178787592207301e-05,
+      "loss": 1.3736,
+      "theoretical_loss": 3.3555313640457776,
+      "tokens_seen": 2624716800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.119680348023454e-05,
+      "loss": 1.3462,
+      "theoretical_loss": 3.355200720293803,
+      "tokens_seen": 2627993600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.060573103839607e-05,
+      "loss": 1.3732,
+      "theoretical_loss": 3.3548706038318867,
+      "tokens_seen": 2631270400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0014658596557594e-05,
+      "loss": 1.3391,
+      "theoretical_loss": 3.3545410131640585,
+      "tokens_seen": 2634547200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9423586154719122e-05,
+      "loss": 1.344,
+      "theoretical_loss": 3.354211946800446,
+      "tokens_seen": 2637824000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8832513712880653e-05,
+      "loss": 1.3191,
+      "theoretical_loss": 3.3538834032572424,
+      "tokens_seen": 2641100800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8241441271042177e-05,
+      "loss": 1.3711,
+      "theoretical_loss": 3.3535553810566743,
+      "tokens_seen": 2644377600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7650368829203708e-05,
+      "loss": 1.3402,
+      "theoretical_loss": 3.3532278787269707,
+      "tokens_seen": 2647654400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.705929638736524e-05,
+      "loss": 1.3331,
+      "theoretical_loss": 3.35290089480233,
+      "tokens_seen": 2650931200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6468223945526763e-05,
+      "loss": 1.3489,
+      "theoretical_loss": 3.352574427822888,
+      "tokens_seen": 2654208000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5877151503688294e-05,
+      "loss": 1.3631,
+      "theoretical_loss": 3.3522484763346885,
+      "tokens_seen": 2657484800
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.529790051068659e-05,
+      "loss": 1.327,
+      "theoretical_loss": 3.3519230388896504,
+      "tokens_seen": 2660761600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4706828068848117e-05,
+      "loss": 1.3363,
+      "theoretical_loss": 3.3515981140455375,
+      "tokens_seen": 2664038400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4115755627009645e-05,
+      "loss": 1.3512,
+      "theoretical_loss": 3.3512737003659288,
+      "tokens_seen": 2667315200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3524683185171176e-05,
+      "loss": 1.3469,
+      "theoretical_loss": 3.350949796420184,
+      "tokens_seen": 2670592000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2933610743332703e-05,
+      "loss": 1.3514,
+      "theoretical_loss": 3.3506264007834186,
+      "tokens_seen": 2673868800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.234253830149423e-05,
+      "loss": 1.371,
+      "theoretical_loss": 3.3503035120364704,
+      "tokens_seen": 2677145600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1751465859655762e-05,
+      "loss": 1.3507,
+      "theoretical_loss": 3.349981128765869,
+      "tokens_seen": 2680422400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1160393417817286e-05,
+      "loss": 1.331,
+      "theoretical_loss": 3.3496592495638096,
+      "tokens_seen": 2683699200
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0569320975978817e-05,
+      "loss": 1.3342,
+      "theoretical_loss": 3.349337873028119,
+      "tokens_seen": 2686976000
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9978248534140345e-05,
+      "loss": 1.3664,
+      "theoretical_loss": 3.349016997762229,
+      "tokens_seen": 2690252800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9387176092301876e-05,
+      "loss": 1.3547,
+      "theoretical_loss": 3.348696622375147,
+      "tokens_seen": 2693529600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.87961036504634e-05,
+      "loss": 1.3679,
+      "theoretical_loss": 3.3483767454814255,
+      "tokens_seen": 2696806400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.820503120862493e-05,
+      "loss": 1.3522,
+      "theoretical_loss": 3.3480573657011354,
+      "tokens_seen": 2700083200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7613958766786458e-05,
+      "loss": 1.3502,
+      "theoretical_loss": 3.347738481659836,
+      "tokens_seen": 2703360000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7022886324947986e-05,
+      "loss": 1.3455,
+      "theoretical_loss": 3.347420091988546,
+      "tokens_seen": 2706636800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6431813883109513e-05,
+      "loss": 1.3689,
+      "theoretical_loss": 3.3471021953237177,
+      "tokens_seen": 2709913600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5840741441271044e-05,
+      "loss": 1.4061,
+      "theoretical_loss": 3.346784790307205,
+      "tokens_seen": 2713190400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.524966899943257e-05,
+      "loss": 1.3691,
+      "theoretical_loss": 3.346467875586239,
+      "tokens_seen": 2716467200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.46585965575941e-05,
+      "loss": 1.3241,
+      "theoretical_loss": 3.3461514498133997,
+      "tokens_seen": 2719744000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4067524115755627e-05,
+      "loss": 1.3172,
+      "theoretical_loss": 3.345835511646587,
+      "tokens_seen": 2723020800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3476451673917156e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.345520059748993,
+      "tokens_seen": 2726297600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2885379232078684e-05,
+      "loss": 1.3288,
+      "theoretical_loss": 3.3452050927890777,
+      "tokens_seen": 2729574400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2294306790240211e-05,
+      "loss": 1.3464,
+      "theoretical_loss": 3.3448906094405393,
+      "tokens_seen": 2732851200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.170323434840174e-05,
+      "loss": 1.3185,
+      "theoretical_loss": 3.344576608382288,
+      "tokens_seen": 2736128000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1112161906563268e-05,
+      "loss": 1.3816,
+      "theoretical_loss": 3.34426308829842,
+      "tokens_seen": 2739404800
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0521089464724796e-05,
+      "loss": 1.363,
+      "theoretical_loss": 3.343950047878189,
+      "tokens_seen": 2742681600
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.930017022886325e-06,
+      "loss": 1.3504,
+      "theoretical_loss": 3.343637485815983,
+      "tokens_seen": 2745958400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.338944581047854e-06,
+      "loss": 1.3662,
+      "theoretical_loss": 3.3433254008112963,
+      "tokens_seen": 2749235200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.747872139209382e-06,
+      "loss": 1.4015,
+      "theoretical_loss": 3.343013791568702,
+      "tokens_seen": 2752512000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.156799697370911e-06,
+      "loss": 1.4013,
+      "theoretical_loss": 3.342702656797829,
+      "tokens_seen": 2755788800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.565727255532439e-06,
+      "loss": 1.3931,
+      "theoretical_loss": 3.342391995213335,
+      "tokens_seen": 2759065600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.974654813693967e-06,
+      "loss": 1.414,
+      "theoretical_loss": 3.34208180553488,
+      "tokens_seen": 2762342400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.383582371855495e-06,
+      "loss": 1.4041,
+      "theoretical_loss": 3.341772086487102,
+      "tokens_seen": 2765619200
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.792509930017023e-06,
+      "loss": 1.4063,
+      "theoretical_loss": 3.341462836799593,
+      "tokens_seen": 2768896000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.2014374881785515e-06,
+      "loss": 1.4269,
+      "theoretical_loss": 3.341154055206871,
+      "tokens_seen": 2772172800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.610365046340079e-06,
+      "loss": 1.433,
+      "theoretical_loss": 3.3408457404483576,
+      "tokens_seen": 2775449600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.0192926045016075e-06,
+      "loss": 1.3909,
+      "theoretical_loss": 3.3405378912683514,
+      "tokens_seen": 2778726400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.4282201626631363e-06,
+      "loss": 1.4514,
+      "theoretical_loss": 3.340230506416005,
+      "tokens_seen": 2782003200
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/aligned_token_count": 0.4079912006855011,
+      "objective/train/avg_token_score": 0.025389162823557854,
+      "objective/train/docs_used": 1566749,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/likelihood_loss": 1.3449466228485107,
+      "objective/train/original_loss": 5.0562591552734375,
+      "objective/train/theoretical_loss": 3.3399235846453004,
+      "objective/train/tokens_used": 2805740000,
+      "objective/train/unlikelihood_loss": 0.04297417774796486,
+      "theoretical_loss": 3.3399235846453004,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.8371477208246643e-06,
+      "loss": 1.4519,
+      "theoretical_loss": 3.3399235846453004,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.2460752789861923e-06,
+      "loss": 1.4355,
+      "theoretical_loss": 3.3396171247150224,
+      "tokens_seen": 2788556800
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6668242859844903e-06,
+      "loss": 1.3904,
+      "theoretical_loss": 3.3393111253887375,
+      "tokens_seen": 2791833600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0757518441460185e-06,
+      "loss": 1.3914,
+      "theoretical_loss": 3.339005585434769,
+      "tokens_seen": 2795110400
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.846794023075469e-07,
+      "loss": 1.3644,
+      "theoretical_loss": 3.338700503626171,
+      "tokens_seen": 2798387200
+    }
+  ],
+  "max_steps": 42724,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.42878663770112e+18,
+  "trial_name": null,
+  "trial_params": null
+}