diff --git "a/checkpoint-42724/trainer_state.json" "b/checkpoint-42724/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-42724/trainer_state.json"
@@ -0,0 +1,6244 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "global_step": 42724,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1682242990654204e-06,
+      "loss": 10.7926,
+      "theoretical_loss": 20.81281176760504,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.841121495327103e-05,
+      "loss": 8.8129,
+      "theoretical_loss": 8.563476630668958,
+      "tokens_seen": 3276800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011682242990654206,
+      "loss": 6.5669,
+      "theoretical_loss": 7.477752684105921,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017523364485981307,
+      "loss": 5.6216,
+      "theoretical_loss": 6.9337484549527915,
+      "tokens_seen": 9830400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023364485981308412,
+      "loss": 5.2369,
+      "theoretical_loss": 6.5835601944843045,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029205607476635517,
+      "loss": 5.0152,
+      "theoretical_loss": 6.3307075311739744,
+      "tokens_seen": 16384000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035046728971962614,
+      "loss": 4.854,
+      "theoretical_loss": 6.135523197998216,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004088785046728972,
+      "loss": 4.7404,
+      "theoretical_loss": 5.978095549927499,
+      "tokens_seen": 22937600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046728971962616824,
+      "loss": 4.6057,
+      "theoretical_loss": 5.847111292323815,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499739928125591,
+      "loss": 4.4738,
+      "theoretical_loss": 5.735570781940016,
+      "tokens_seen": 29491200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991488556837526,
+      "loss": 4.3968,
+      "theoretical_loss": 5.638864110129244,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985577832419141,
+      "loss": 4.2438,
+      "theoretical_loss": 5.553806347902798,
+      "tokens_seen": 36044800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979667108000757,
+      "loss": 4.1836,
+      "theoretical_loss": 5.478112046614329,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973756383582371,
+      "loss": 4.1064,
+      "theoretical_loss": 5.410089925637252,
+      "tokens_seen": 42598400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967845659163987,
+      "loss": 4.0626,
+      "theoretical_loss": 5.348456049793725,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961934934745603,
+      "loss": 3.9378,
+      "theoretical_loss": 5.292214532995457,
+      "tokens_seen": 49152000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956024210327218,
+      "loss": 3.9244,
+      "theoretical_loss": 5.240578591827869,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950113485908833,
+      "loss": 3.8394,
+      "theoretical_loss": 5.192916690583679,
+      "tokens_seen": 55705600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944202761490448,
+      "loss": 3.761,
+      "theoretical_loss": 5.148714829414331,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938292037072064,
+      "loss": 3.6902,
+      "theoretical_loss": 5.107549528462992,
+      "tokens_seen": 62259200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932381312653678,
+      "loss": 3.649,
+      "theoretical_loss": 5.069068083201136,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926470588235294,
+      "loss": 3.5367,
+      "theoretical_loss": 5.032973875895897,
+      "tokens_seen": 68812800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492055986381691,
+      "loss": 3.473,
+      "theoretical_loss": 4.999015274282555,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914649139398525,
+      "loss": 3.3792,
+      "theoretical_loss": 4.966977121409853,
+      "tokens_seen": 75366400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490873841498014,
+      "loss": 3.3265,
+      "theoretical_loss": 4.936674127683027,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902827690561755,
+      "loss": 3.3293,
+      "theoretical_loss": 4.907945679887972,
+      "tokens_seen": 81920000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896916966143371,
+      "loss": 3.321,
+      "theoretical_loss": 4.8806517198708175,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891006241724985,
+      "loss": 3.2217,
+      "theoretical_loss": 4.85466944053967,
+      "tokens_seen": 88473600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885095517306601,
+      "loss": 3.2277,
+      "theoretical_loss": 4.829890613366704,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879184792888217,
+      "loss": 3.2068,
+      "theoretical_loss": 4.806219408835812,
+      "tokens_seen": 95027200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873274068469832,
+      "loss": 3.1622,
+      "theoretical_loss": 4.783570605334148,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048673633440514467,
+      "loss": 3.1879,
+      "theoretical_loss": 4.761868106830299,
+      "tokens_seen": 101580800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861452619633062,
+      "loss": 3.1352,
+      "theoretical_loss": 4.741043708020364,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855541895214677,
+      "loss": 3.1048,
+      "theoretical_loss": 4.721036059306941,
+      "tokens_seen": 108134400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849631170796293,
+      "loss": 3.1318,
+      "theoretical_loss": 4.701789794289756,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843720446377908,
+      "loss": 3.1115,
+      "theoretical_loss": 4.68325479029382,
+      "tokens_seen": 114688000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048378097219595233,
+      "loss": 3.1102,
+      "theoretical_loss": 4.6653855384841725,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048318989975411385,
+      "loss": 3.0769,
+      "theoretical_loss": 4.6481406047776295,
+      "tokens_seen": 121241600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048259882731227537,
+      "loss": 3.0882,
+      "theoretical_loss": 4.631482166397534,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820077548704369,
+      "loss": 3.084,
+      "theoretical_loss": 4.615375611773608,
+      "tokens_seen": 127795200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141668242859847,
+      "loss": 3.0752,
+      "theoretical_loss": 4.5997891937483955,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048082560998676,
+      "loss": 3.0295,
+      "theoretical_loss": 4.584693727850565,
+      "tokens_seen": 134348800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802345375449215,
+      "loss": 3.0131,
+      "theoretical_loss": 4.570062328836407,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047964346510308303,
+      "loss": 3.0004,
+      "theoretical_loss": 4.5558701798619285,
+      "tokens_seen": 140902400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905239266124455,
+      "loss": 3.0324,
+      "theoretical_loss": 4.542094329588689,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047846132021940607,
+      "loss": 3.0272,
+      "theoretical_loss": 4.528713513292708,
+      "tokens_seen": 147456000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778702477775676,
+      "loss": 2.9858,
+      "theoretical_loss": 4.515707994672887,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047727917533572917,
+      "loss": 2.9764,
+      "theoretical_loss": 4.503059425571229,
+      "tokens_seen": 154009600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766881028938907,
+      "loss": 2.9751,
+      "theoretical_loss": 4.490750721243157,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760970304520522,
+      "loss": 3.005,
+      "theoretical_loss": 4.478765949169858,
+      "tokens_seen": 160563200
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 91708,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.834399700164795,
+      "objective/train/theoretical_loss": 4.46709022969911,
+      "objective/train/tokens_used": 184300000,
+      "theoretical_loss": 4.46709022969911,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047550595801021373,
+      "loss": 2.9681,
+      "theoretical_loss": 4.46709022969911,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047491488556837525,
+      "loss": 2.9537,
+      "theoretical_loss": 4.455709647047437,
+      "tokens_seen": 167116800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432381312653677,
+      "loss": 2.9301,
+      "theoretical_loss": 4.444611169403321,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047373274068469835,
+      "loss": 2.9362,
+      "theoretical_loss": 4.4337825770455375,
+      "tokens_seen": 173670400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047314166824285987,
+      "loss": 2.9216,
+      "theoretical_loss": 4.423212397538051,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725505958010214,
+      "loss": 2.9052,
+      "theoretical_loss": 4.4128898471880325,
+      "tokens_seen": 180224000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719595233591829,
+      "loss": 2.9085,
+      "theoretical_loss": 4.40280477805997,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047136845091734443,
+      "loss": 2.929,
+      "theoretical_loss": 4.392947629929752,
+      "tokens_seen": 186777600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047077737847550595,
+      "loss": 2.934,
+      "theoretical_loss": 4.383309386640423,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701863060336675,
+      "loss": 2.909,
+      "theoretical_loss": 4.373881536388167,
+      "tokens_seen": 193331200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046959523359182905,
+      "loss": 2.8757,
+      "theoretical_loss": 4.364656035524595,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046900416114999057,
+      "loss": 2.8927,
+      "theoretical_loss": 4.355625275511174,
+      "tokens_seen": 199884800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684130887081521,
+      "loss": 2.8711,
+      "theoretical_loss": 4.346782052704563,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678220162663136,
+      "loss": 2.8508,
+      "theoretical_loss": 4.338119540689052,
+      "tokens_seen": 206438400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046723094382447513,
+      "loss": 2.8683,
+      "theoretical_loss": 4.329631264904703,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046663987138263665,
+      "loss": 2.8571,
+      "theoretical_loss": 4.321311079348144,
+      "tokens_seen": 212992000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660487989407982,
+      "loss": 2.888,
+      "theoretical_loss": 4.313153145147683,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046545772649895975,
+      "loss": 2.8459,
+      "theoretical_loss": 4.305151910836119,
+      "tokens_seen": 219545600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046486665405712127,
+      "loss": 2.8228,
+      "theoretical_loss": 4.2973020941635784,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642755816152828,
+      "loss": 2.8308,
+      "theoretical_loss": 4.28959866530949,
+      "tokens_seen": 226099200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636845091734443,
+      "loss": 2.8282,
+      "theoretical_loss": 4.282036831367506,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046309343673160583,
+      "loss": 2.8328,
+      "theoretical_loss": 4.274612021990189,
+      "tokens_seen": 232652800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625023642897674,
+      "loss": 2.859,
+      "theoretical_loss": 4.267319876091788,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619112918479289,
+      "loss": 2.8342,
+      "theoretical_loss": 4.260156229517635,
+      "tokens_seen": 239206400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046132021940609044,
+      "loss": 2.7766,
+      "theoretical_loss": 4.253117103597704,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046072914696425197,
+      "loss": 2.7575,
+      "theoretical_loss": 4.246198694509945,
+      "tokens_seen": 245760000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601380745224135,
+      "loss": 2.7936,
+      "theoretical_loss": 4.239397363386152,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459547002080575,
+      "loss": 2.8022,
+      "theoretical_loss": 4.232709627099522,
+      "tokens_seen": 252313600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589559296387366,
+      "loss": 2.8341,
+      "theoretical_loss": 4.226132149678757,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583648571968981,
+      "loss": 2.8652,
+      "theoretical_loss": 4.219661734298666,
+      "tokens_seen": 258867200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577737847550596,
+      "loss": 2.849,
+      "theoretical_loss": 4.213295315801815,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571827123132211,
+      "loss": 2.8169,
+      "theoretical_loss": 4.207029953709861,
+      "tokens_seen": 265420800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565916398713826,
+      "loss": 2.8278,
+      "theoretical_loss": 4.200862825686893,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045600056742954413,
+      "loss": 2.7814,
+      "theoretical_loss": 4.19479122142044,
+      "tokens_seen": 271974400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554094949877057,
+      "loss": 2.7896,
+      "theoretical_loss": 4.188812536888775,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045481842254586723,
+      "loss": 2.7571,
+      "theoretical_loss": 4.182924268985855,
+      "tokens_seen": 278528000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045422735010402875,
+      "loss": 2.7364,
+      "theoretical_loss": 4.177124010477671,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045363627766219027,
+      "loss": 2.7741,
+      "theoretical_loss": 4.171409445265983,
+      "tokens_seen": 285081600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004530452052203518,
+      "loss": 2.7755,
+      "theoretical_loss": 4.165778343937409,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004524541327785133,
+      "loss": 2.801,
+      "theoretical_loss": 4.160228559577659,
+      "tokens_seen": 291635200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045186306033667483,
+      "loss": 2.761,
+      "theoretical_loss": 4.15475802383233,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004512719878948364,
+      "loss": 2.6851,
+      "theoretical_loss": 4.149364743197177,
+      "tokens_seen": 298188800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045068091545299793,
+      "loss": 2.7103,
+      "theoretical_loss": 4.14404679552214,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045008984301115945,
+      "loss": 2.6929,
+      "theoretical_loss": 4.138802326714632,
+      "tokens_seen": 304742400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044949877056932097,
+      "loss": 2.6886,
+      "theoretical_loss": 4.133629547628726,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004489076981274825,
+      "loss": 2.6871,
+      "theoretical_loss": 4.128526731127894,
+      "tokens_seen": 311296000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000448316625685644,
+      "loss": 2.7051,
+      "theoretical_loss": 4.123492209309923,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004477255532438056,
+      "loss": 2.7244,
+      "theoretical_loss": 4.118524370883447,
+      "tokens_seen": 317849600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471344808019671,
+      "loss": 2.7389,
+      "theoretical_loss": 4.113621658686355,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044654340836012863,
+      "loss": 2.7728,
+      "theoretical_loss": 4.108782567337039,
+      "tokens_seen": 324403200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6607348357914675,
+      "debugging/distinct-1-grams": 0.8023495471425544,
+      "debugging/distinct-2-grams": 0.9518089508119704,
+      "debugging/entropy-1-grams": 6.089592994551723,
+      "debugging/entropy-2-grams": 6.90484015923658,
+      "debugging/length": 683.3888888888889,
+      "debugging/num_segments": 18,
+      "epoch": 0.12,
+      "objective/train/docs_used": 173345,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.669816255569458,
+      "objective/train/theoretical_loss": 4.104005641010112,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.104005641010112,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044595233591829015,
+      "loss": 2.7658,
+      "theoretical_loss": 4.104005641010112,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044536126347645167,
+      "loss": 2.7501,
+      "theoretical_loss": 4.099289471328812,
+      "tokens_seen": 330956800
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004447701910346132,
+      "loss": 2.7384,
+      "theoretical_loss": 4.094632695366921,
+      "tokens_seen": 334233600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044417911859277476,
+      "loss": 2.7302,
+      "theoretical_loss": 4.090033993753448,
+      "tokens_seen": 337510400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435880461509363,
+      "loss": 2.6847,
+      "theoretical_loss": 4.085492088873883,
+      "tokens_seen": 340787200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004429969737090978,
+      "loss": 2.6919,
+      "theoretical_loss": 4.081005743162224,
+      "tokens_seen": 344064000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004424059012672593,
+      "loss": 2.6785,
+      "theoretical_loss": 4.076573757478361,
+      "tokens_seen": 347340800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044181482882542085,
+      "loss": 2.667,
+      "theoretical_loss": 4.072194969565807,
+      "tokens_seen": 350617600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044122375638358237,
+      "loss": 2.6537,
+      "theoretical_loss": 4.067868252585089,
+      "tokens_seen": 353894400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004406326839417439,
+      "loss": 2.669,
+      "theoretical_loss": 4.063592513718411,
+      "tokens_seen": 357171200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044004161149990546,
+      "loss": 2.6391,
+      "theoretical_loss": 4.059366692841521,
+      "tokens_seen": 360448000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000439450539058067,
+      "loss": 2.638,
+      "theoretical_loss": 4.055189761258959,
+      "tokens_seen": 363724800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388594666162285,
+      "loss": 2.6594,
+      "theoretical_loss": 4.051060720499127,
+      "tokens_seen": 367001600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043826839417439,
+      "loss": 2.6626,
+      "theoretical_loss": 4.046978601165831,
+      "tokens_seen": 370278400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043767732173255155,
+      "loss": 2.6763,
+      "theoretical_loss": 4.042942461843204,
+      "tokens_seen": 373555200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043708624929071307,
+      "loss": 2.6615,
+      "theoretical_loss": 4.038951388051044,
+      "tokens_seen": 376832000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043649517684887464,
+      "loss": 2.6649,
+      "theoretical_loss": 4.035004491247873,
+      "tokens_seen": 380108800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043590410440703616,
+      "loss": 2.6609,
+      "theoretical_loss": 4.031100907879109,
+      "tokens_seen": 383385600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004353130319651977,
+      "loss": 2.6541,
+      "theoretical_loss": 4.02723979846797,
+      "tokens_seen": 386662400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004347219595233592,
+      "loss": 2.6331,
+      "theoretical_loss": 4.023420346746835,
+      "tokens_seen": 389939200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004341308870815207,
+      "loss": 2.6643,
+      "theoretical_loss": 4.019641758826938,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043353981463968225,
+      "loss": 2.7007,
+      "theoretical_loss": 4.015903262404413,
+      "tokens_seen": 396492800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004329487421978438,
+      "loss": 2.6606,
+      "theoretical_loss": 4.012204106000786,
+      "tokens_seen": 399769600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043235766975600534,
+      "loss": 2.6741,
+      "theoretical_loss": 4.008543558236181,
+      "tokens_seen": 403046400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043176659731416686,
+      "loss": 2.6851,
+      "theoretical_loss": 4.004920907133565,
+      "tokens_seen": 406323200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004311755248723284,
+      "loss": 2.6507,
+      "theoretical_loss": 4.001335459452449,
+      "tokens_seen": 409600000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004305844524304899,
+      "loss": 2.6388,
+      "theoretical_loss": 3.997786540050617,
+      "tokens_seen": 412876800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004299933799886514,
+      "loss": 2.6775,
+      "theoretical_loss": 3.9942734912724456,
+      "tokens_seen": 416153600
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042940230754681295,
+      "loss": 2.6459,
+      "theoretical_loss": 3.9907956723625375,
+      "tokens_seen": 419430400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004288112351049745,
+      "loss": 2.6205,
+      "theoretical_loss": 3.9873524589034224,
+      "tokens_seen": 422707200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000428220162663136,
+      "loss": 2.6119,
+      "theoretical_loss": 3.9839432422761556,
+      "tokens_seen": 425984000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004276290902212975,
+      "loss": 2.5946,
+      "theoretical_loss": 3.980567429142721,
+      "tokens_seen": 429260800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042703801777945903,
+      "loss": 2.6213,
+      "theoretical_loss": 3.977224440949197,
+      "tokens_seen": 432537600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042644694533762055,
+      "loss": 2.6306,
+      "theoretical_loss": 3.9739137134486917,
+      "tokens_seen": 435814400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042585587289578207,
+      "loss": 2.621,
+      "theoretical_loss": 3.9706346962431396,
+      "tokens_seen": 439091200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042526480045394365,
+      "loss": 2.6262,
+      "theoretical_loss": 3.9673868523430564,
+      "tokens_seen": 442368000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042467372801210517,
+      "loss": 2.6389,
+      "theoretical_loss": 3.9641696577444376,
+      "tokens_seen": 445644800
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004240826555702667,
+      "loss": 2.6321,
+      "theoretical_loss": 3.9609826010220033,
+      "tokens_seen": 448921600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004234915831284282,
+      "loss": 2.6313,
+      "theoretical_loss": 3.9578251829380506,
+      "tokens_seen": 452198400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042290051068658973,
+      "loss": 2.6319,
+      "theoretical_loss": 3.954696916066199,
+      "tokens_seen": 455475200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042230943824475125,
+      "loss": 2.6369,
+      "theoretical_loss": 3.9515973244293643,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004217183658029128,
+      "loss": 2.648,
+      "theoretical_loss": 3.948525943151326,
+      "tokens_seen": 462028800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042112729336107435,
+      "loss": 2.674,
+      "theoretical_loss": 3.9454823181212815,
+      "tokens_seen": 465305600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042053622091923587,
+      "loss": 2.6931,
+      "theoretical_loss": 3.9424660056708167,
+      "tokens_seen": 468582400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199451484773974,
+      "loss": 2.6383,
+      "theoretical_loss": 3.939476572262754,
+      "tokens_seen": 471859200
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004193540760355589,
+      "loss": 2.6659,
+      "theoretical_loss": 3.9365135941913563,
+      "tokens_seen": 475136000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041876300359372043,
+      "loss": 2.6818,
+      "theoretical_loss": 3.9335766572934023,
+      "tokens_seen": 478412800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041817193115188195,
+      "loss": 2.6567,
+      "theoretical_loss": 3.9306653566696603,
+      "tokens_seen": 481689600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004175808587100435,
+      "loss": 2.6558,
+      "theoretical_loss": 3.927779296416332,
+      "tokens_seen": 484966400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041698978626820505,
+      "loss": 2.6617,
+      "theoretical_loss": 3.924918089366024,
+      "tokens_seen": 488243200
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 254462,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4227302074432373,
+      "objective/train/theoretical_loss": 3.9220813568378707,
+      "objective/train/tokens_used": 511980000,
+      "theoretical_loss": 3.9220813568378707,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041639871382636657,
+      "loss": 2.658,
+      "theoretical_loss": 3.9220813568378707,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004158076413845281,
+      "loss": 2.5929,
+      "theoretical_loss": 3.9192687283964096,
+      "tokens_seen": 494796800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041522839039152636,
+      "loss": 2.6373,
+      "theoretical_loss": 3.9164798416188527,
+      "tokens_seen": 498073600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004146373179496879,
+      "loss": 2.6258,
+      "theoretical_loss": 3.913714341870409,
+      "tokens_seen": 501350400
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041404624550784946,
+      "loss": 2.6303,
+      "theoretical_loss": 3.9109718820873303,
+      "tokens_seen": 504627200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000413455173066011,
+      "loss": 2.6529,
+      "theoretical_loss": 3.9082521225673625,
+      "tokens_seen": 507904000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004128641006241725,
+      "loss": 2.6082,
+      "theoretical_loss": 3.9055547307673075,
+      "tokens_seen": 511180800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000412273028182334,
+      "loss": 2.5831,
+      "theoretical_loss": 3.9028793811074056,
+      "tokens_seen": 514457600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041168195574049554,
+      "loss": 2.6007,
+      "theoretical_loss": 3.900225754782274,
+      "tokens_seen": 517734400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041109088329865706,
+      "loss": 2.596,
+      "theoretical_loss": 3.897593539578138,
+      "tokens_seen": 521011200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041049981085681863,
+      "loss": 2.5878,
+      "theoretical_loss": 3.8949824296961015,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040990873841498016,
+      "loss": 2.5884,
+      "theoretical_loss": 3.8923921255812353,
+      "tokens_seen": 527564800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004093176659731417,
+      "loss": 2.5497,
+      "theoretical_loss": 3.8898223337572393,
+      "tokens_seen": 530841600
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004087265935313032,
+      "loss": 2.571,
+      "theoretical_loss": 3.88727276666648,
+      "tokens_seen": 534118400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004081355210894647,
+      "loss": 2.5724,
+      "theoretical_loss": 3.884743142515184,
+      "tokens_seen": 537395200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040754444864762624,
+      "loss": 2.5607,
+      "theoretical_loss": 3.8822331851235985,
+      "tokens_seen": 540672000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004069533762057878,
+      "loss": 2.6168,
+      "theoretical_loss": 3.87974262378093,
+      "tokens_seen": 543948800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040636230376394933,
+      "loss": 2.6095,
+      "theoretical_loss": 3.877271193104873,
+      "tokens_seen": 547225600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040577123132211085,
+      "loss": 2.5993,
+      "theoretical_loss": 3.8748186329055736,
+      "tokens_seen": 550502400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004051801588802724,
+      "loss": 2.575,
+      "theoretical_loss": 3.87238468805384,
+      "tokens_seen": 553779200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004045890864384339,
+      "loss": 2.5741,
+      "theoretical_loss": 3.8699691083534633,
+      "tokens_seen": 557056000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004039980139965954,
+      "loss": 2.5787,
+      "theoretical_loss": 3.8675716484174907,
+      "tokens_seen": 560332800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040340694155475694,
+      "loss": 2.5681,
+      "theoretical_loss": 3.8651920675482936,
+      "tokens_seen": 563609600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004028158691129185,
+      "loss": 2.5427,
+      "theoretical_loss": 3.862830129621318,
+      "tokens_seen": 566886400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040222479667108003,
+      "loss": 2.5492,
+      "theoretical_loss": 3.8604856029723575,
+      "tokens_seen": 570163200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040163372422924155,
+      "loss": 2.5492,
+      "theoretical_loss": 3.8581582602882447,
+      "tokens_seen": 573440000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004010426517874031,
+      "loss": 2.526,
+      "theoretical_loss": 3.8558478785008203,
+      "tokens_seen": 576716800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004004515793455646,
+      "loss": 2.5441,
+      "theoretical_loss": 3.8535542386840778,
+      "tokens_seen": 579993600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003998605069037261,
+      "loss": 2.5473,
+      "theoretical_loss": 3.8512771259543586,
+      "tokens_seen": 583270400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003992694344618877,
+      "loss": 2.5235,
+      "theoretical_loss": 3.8490163293735082,
+      "tokens_seen": 586547200
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003986783620200492,
+      "loss": 2.5408,
+      "theoretical_loss": 3.8467716418548648,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039808728957821073,
+      "loss": 2.5857,
+      "theoretical_loss": 3.844542860072007,
+      "tokens_seen": 593100800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039749621713637225,
+      "loss": 2.5715,
+      "theoretical_loss": 3.8423297843701496,
+      "tokens_seen": 596377600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003969051446945338,
+      "loss": 2.5891,
+      "theoretical_loss": 3.8401322186800995,
+      "tokens_seen": 599654400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003963140722526953,
+      "loss": 2.5636,
+      "theoretical_loss": 3.83794997043469,
+      "tokens_seen": 602931200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039572299981085687,
+      "loss": 2.5828,
+      "theoretical_loss": 3.8357828504876004,
+      "tokens_seen": 606208000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003951319273690184,
+      "loss": 2.5936,
+      "theoretical_loss": 3.833630673034487,
+      "tokens_seen": 609484800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003945408549271799,
+      "loss": 2.5763,
+      "theoretical_loss": 3.831493255536345,
+      "tokens_seen": 612761600
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039394978248534143,
+      "loss": 2.5747,
+      "theoretical_loss": 3.8293704186450253,
+      "tokens_seen": 616038400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039335871004350295,
+      "loss": 2.596,
+      "theoretical_loss": 3.827261986130839,
+      "tokens_seen": 619315200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003927676376016645,
+      "loss": 2.5675,
+      "theoretical_loss": 3.825167784812175,
+      "tokens_seen": 622592000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000392176565159826,
+      "loss": 2.5751,
+      "theoretical_loss": 3.823087644487069,
+      "tokens_seen": 625868800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039158549271798757,
+      "loss": 2.6107,
+      "theoretical_loss": 3.8210213978666565,
+      "tokens_seen": 629145600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003909944202761491,
+      "loss": 2.6032,
+      "theoretical_loss": 3.8189688805104476,
+      "tokens_seen": 632422400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039040334783431056,
+      "loss": 2.6353,
+      "theoretical_loss": 3.816929930763374,
+      "tokens_seen": 635699200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003898122753924721,
+      "loss": 2.6048,
+      "theoretical_loss": 3.8149043896945347,
+      "tokens_seen": 638976000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003892212029506336,
+      "loss": 2.5951,
+      "theoretical_loss": 3.812892101037601,
+      "tokens_seen": 642252800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003886301305087951,
+      "loss": 2.6095,
+      "theoretical_loss": 3.81089291113282,
+      "tokens_seen": 645529600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003880390580669567,
+      "loss": 2.5981,
+      "theoretical_loss": 3.8089066688705673,
+      "tokens_seen": 648806400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003874479856251182,
+      "loss": 2.5641,
+      "theoretical_loss": 3.8069332256363992,
+      "tokens_seen": 652083200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5613787023201813,
+      "debugging/distinct-1-grams": 0.8149083264126824,
+      "debugging/distinct-2-grams": 0.9601495159951162,
+      "debugging/entropy-1-grams": 5.947486888431812,
+      "debugging/entropy-2-grams": 6.555422311272144,
+      "debugging/length": 621.5384615384615,
+      "debugging/num_segments": 13,
+      "epoch": 0.23,
+      "objective/train/docs_used": 335300,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.533392906188965,
+      "objective/train/theoretical_loss": 3.80497243525756,
+      "objective/train/tokens_used": 675820000,
+      "theoretical_loss": 3.80497243525756,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038685691318327974,
+      "loss": 2.5531,
+      "theoretical_loss": 3.80497243525756,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038626584074144126,
+      "loss": 2.5586,
+      "theoretical_loss": 3.8030241539508958,
+      "tokens_seen": 658636800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003856747682996028,
+      "loss": 2.5483,
+      "theoretical_loss": 3.8010882402721324,
+      "tokens_seen": 661913600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003850836958577643,
+      "loss": 2.5549,
+      "theoretical_loss": 3.7991645550664757,
+      "tokens_seen": 665190400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003844926234159259,
+      "loss": 2.5372,
+      "theoretical_loss": 3.797252961420492,
+      "tokens_seen": 668467200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003839015509740874,
+      "loss": 2.5367,
+      "theoretical_loss": 3.795353324615228,
+      "tokens_seen": 671744000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833104785322489,
+      "loss": 2.5248,
+      "theoretical_loss": 3.793465512080541,
+      "tokens_seen": 675020800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038271940609041044,
+      "loss": 2.5511,
+      "theoretical_loss": 3.791589393350587,
+      "tokens_seen": 678297600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038212833364857196,
+      "loss": 2.5481,
+      "theoretical_loss": 3.7897248400204475,
+      "tokens_seen": 681574400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003815372612067335,
+      "loss": 2.541,
+      "theoretical_loss": 3.7878717257038534,
+      "tokens_seen": 684851200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000380946188764895,
+      "loss": 2.5462,
+      "theoretical_loss": 3.7860299259919685,
+      "tokens_seen": 688128000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003803551163230566,
+      "loss": 2.5154,
+      "theoretical_loss": 3.7841993184132114,
+      "tokens_seen": 691404800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003797640438812181,
+      "loss": 2.5001,
+      "theoretical_loss": 3.78237978239408,
+      "tokens_seen": 694681600
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003791729714393796,
+      "loss": 2.5389,
+      "theoretical_loss": 3.780571199220942,
+      "tokens_seen": 697958400
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037858189899754114,
+      "loss": 2.5683,
+      "theoretical_loss": 3.7787734520027803,
+      "tokens_seen": 701235200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037799082655570266,
+      "loss": 2.5869,
+      "theoretical_loss": 3.7769864256348455,
+      "tokens_seen": 704512000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003773997541138642,
+      "loss": 2.6073,
+      "theoretical_loss": 3.775210006763202,
+      "tokens_seen": 707788800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037680868167202575,
+      "loss": 2.5991,
+      "theoretical_loss": 3.7734440837501406,
+      "tokens_seen": 711065600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003762176092301873,
+      "loss": 2.5618,
+      "theoretical_loss": 3.7716885466404246,
+      "tokens_seen": 714342400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003756265367883488,
+      "loss": 2.5421,
+      "theoretical_loss": 3.769943287128357,
+      "tokens_seen": 717619200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003750354643465103,
+      "loss": 2.5525,
+      "theoretical_loss": 3.7682081985256364,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037444439190467184,
+      "loss": 2.5112,
+      "theoretical_loss": 3.7664831757299795,
+      "tokens_seen": 724172800
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037385331946283336,
+      "loss": 2.5343,
+      "theoretical_loss": 3.7647681151944976,
+      "tokens_seen": 727449600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037326224702099493,
+      "loss": 2.5372,
+      "theoretical_loss": 3.7630629148977937,
+      "tokens_seen": 730726400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037267117457915645,
+      "loss": 2.5665,
+      "theoretical_loss": 3.761367474314768,
+      "tokens_seen": 734003200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000372080102137318,
+      "loss": 2.5266,
+      "theoretical_loss": 3.7596816943881084,
+      "tokens_seen": 737280000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003714890296954795,
+      "loss": 2.585,
+      "theoretical_loss": 3.758005477500451,
+      "tokens_seen": 740556800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000370897957253641,
+      "loss": 2.5451,
+      "theoretical_loss": 3.756338727447186,
+      "tokens_seen": 743833600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037030688481180254,
+      "loss": 2.5604,
+      "theoretical_loss": 3.7546813494098945,
+      "tokens_seen": 747110400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036971581236996406,
+      "loss": 2.5853,
+      "theoretical_loss": 3.7530332499304007,
+      "tokens_seen": 750387200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003691365613769624,
+      "loss": 2.5897,
+      "theoretical_loss": 3.7513943368854195,
+      "tokens_seen": 753664000
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003685454889351239,
+      "loss": 2.599,
+      "theoretical_loss": 3.7497645194617863,
+      "tokens_seen": 756940800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679544164932854,
+      "loss": 2.5786,
+      "theoretical_loss": 3.748143708132246,
+      "tokens_seen": 760217600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036736334405144695,
+      "loss": 2.5257,
+      "theoretical_loss": 3.7465318146317994,
+      "tokens_seen": 763494400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036677227160960847,
+      "loss": 2.5269,
+      "theoretical_loss": 3.7449287519345766,
+      "tokens_seen": 766771200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036618119916777,
+      "loss": 2.5517,
+      "theoretical_loss": 3.7433344342312385,
+      "tokens_seen": 770048000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036559012672593156,
+      "loss": 2.5221,
+      "theoretical_loss": 3.7417487769068756,
+      "tokens_seen": 773324800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003649990542840931,
+      "loss": 2.4986,
+      "theoretical_loss": 3.7401716965194076,
+      "tokens_seen": 776601600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003644079818422546,
+      "loss": 2.5261,
+      "theoretical_loss": 3.738603110778461,
+      "tokens_seen": 779878400
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003638169094004161,
+      "loss": 2.5152,
+      "theoretical_loss": 3.73704293852471,
+      "tokens_seen": 783155200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036322583695857765,
+      "loss": 2.5525,
+      "theoretical_loss": 3.7354910997096793,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036263476451673917,
+      "loss": 2.5747,
+      "theoretical_loss": 3.7339475153759825,
+      "tokens_seen": 789708800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036204369207490074,
+      "loss": 2.5238,
+      "theoretical_loss": 3.732412107638,
+      "tokens_seen": 792985600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036145261963306226,
+      "loss": 2.4792,
+      "theoretical_loss": 3.7308847996629724,
+      "tokens_seen": 796262400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003608615471912238,
+      "loss": 2.4976,
+      "theoretical_loss": 3.7293655156525043,
+      "tokens_seen": 799539200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003602704747493853,
+      "loss": 2.4627,
+      "theoretical_loss": 3.727854180824469,
+      "tokens_seen": 802816000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003596794023075468,
+      "loss": 2.4573,
+      "theoretical_loss": 3.7263507213952978,
+      "tokens_seen": 806092800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035908832986570834,
+      "loss": 2.4972,
+      "theoretical_loss": 3.724855064562658,
+      "tokens_seen": 809369600
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003584972574238699,
+      "loss": 2.487,
+      "theoretical_loss": 3.723367138488488,
+      "tokens_seen": 812646400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035790618498203144,
+      "loss": 2.5415,
+      "theoretical_loss": 3.7218868722824014,
+      "tokens_seen": 815923200
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 416670,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7136542797088623,
+      "objective/train/theoretical_loss": 3.7204141959854384,
+      "objective/train/tokens_used": 839660000,
+      "theoretical_loss": 3.7204141959854384,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035731511254019296,
+      "loss": 2.512,
+      "theoretical_loss": 3.7204141959854384,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003567240400983545,
+      "loss": 2.5015,
+      "theoretical_loss": 3.718949040554162,
+      "tokens_seen": 822476800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000356132967656516,
+      "loss": 2.5342,
+      "theoretical_loss": 3.7174913378450833,
+      "tokens_seen": 825753600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003555418952146775,
+      "loss": 2.5043,
+      "theoretical_loss": 3.7160410205994183,
+      "tokens_seen": 829030400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035495082277283904,
+      "loss": 2.477,
+      "theoretical_loss": 3.7145980224281585,
+      "tokens_seen": 832307200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003543597503310006,
+      "loss": 2.4869,
+      "theoretical_loss": 3.713162277797449,
+      "tokens_seen": 835584000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035376867788916214,
+      "loss": 2.4897,
+      "theoretical_loss": 3.7117337220142748,
+      "tokens_seen": 838860800
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035317760544732366,
+      "loss": 2.502,
+      "theoretical_loss": 3.7103122912124364,
+      "tokens_seen": 842137600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003525865330054852,
+      "loss": 2.456,
+      "theoretical_loss": 3.7088979223388128,
+      "tokens_seen": 845414400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035199546056364665,
+      "loss": 2.4775,
+      "theoretical_loss": 3.70749055313991,
+      "tokens_seen": 848691200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035140438812180817,
+      "loss": 2.4525,
+      "theoretical_loss": 3.7060901221486766,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035081331567996974,
+      "loss": 2.4939,
+      "theoretical_loss": 3.704696568671591,
+      "tokens_seen": 855244800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035022224323813127,
+      "loss": 2.4979,
+      "theoretical_loss": 3.7033098327760063,
+      "tokens_seen": 858521600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003496311707962928,
+      "loss": 2.4961,
+      "theoretical_loss": 3.7019298552777533,
+      "tokens_seen": 861798400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003490400983544543,
+      "loss": 2.517,
+      "theoretical_loss": 3.700556577728988,
+      "tokens_seen": 865075200
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034844902591261583,
+      "loss": 2.5434,
+      "theoretical_loss": 3.6991899424062815,
+      "tokens_seen": 868352000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034785795347077735,
+      "loss": 2.5203,
+      "theoretical_loss": 3.697829892298951,
+      "tokens_seen": 871628800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003472668810289389,
+      "loss": 2.5336,
+      "theoretical_loss": 3.696476371097618,
+      "tokens_seen": 874905600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034667580858710044,
+      "loss": 2.5414,
+      "theoretical_loss": 3.695129323182993,
+      "tokens_seen": 878182400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034608473614526196,
+      "loss": 2.5387,
+      "theoretical_loss": 3.693788693614879,
+      "tokens_seen": 881459200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003454936637034235,
+      "loss": 2.564,
+      "theoretical_loss": 3.6924544281213967,
+      "tokens_seen": 884736000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000344902591261585,
+      "loss": 2.5277,
+      "theoretical_loss": 3.691126473088412,
+      "tokens_seen": 888012800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034431151881974653,
+      "loss": 2.5106,
+      "theoretical_loss": 3.689804775549173,
+      "tokens_seen": 891289600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034372044637790805,
+      "loss": 2.535,
+      "theoretical_loss": 3.688489283174146,
+      "tokens_seen": 894566400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003431293739360696,
+      "loss": 2.5195,
+      "theoretical_loss": 3.6871799442610538,
+      "tokens_seen": 897843200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034253830149423114,
+      "loss": 2.5172,
+      "theoretical_loss": 3.685876707725093,
+      "tokens_seen": 901120000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034194722905239266,
+      "loss": 2.542,
+      "theoretical_loss": 3.6845795230893517,
+      "tokens_seen": 904396800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003413561566105542,
+      "loss": 2.5073,
+      "theoretical_loss": 3.6832883404754035,
+      "tokens_seen": 907673600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034077690561755246,
+      "loss": 2.5154,
+      "theoretical_loss": 3.6820031105940796,
+      "tokens_seen": 910950400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000340185833175714,
+      "loss": 2.5224,
+      "theoretical_loss": 3.6807237847364176,
+      "tokens_seen": 914227200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033959476073387555,
+      "loss": 2.5197,
+      "theoretical_loss": 3.6794503147647846,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003390036882920371,
+      "loss": 2.4806,
+      "theoretical_loss": 3.67818265310416,
+      "tokens_seen": 920780800
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003384126158501986,
+      "loss": 2.4864,
+      "theoretical_loss": 3.6769207527335888,
+      "tokens_seen": 924057600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003378215434083601,
+      "loss": 2.4467,
+      "theoretical_loss": 3.675664567177787,
+      "tokens_seen": 927334400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033723047096652164,
+      "loss": 2.448,
+      "theoretical_loss": 3.674414050498913,
+      "tokens_seen": 930611200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033663939852468316,
+      "loss": 2.4737,
+      "theoretical_loss": 3.6731691572884824,
+      "tokens_seen": 933888000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033604832608284473,
+      "loss": 2.4584,
+      "theoretical_loss": 3.671929842659438,
+      "tokens_seen": 937164800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033545725364100625,
+      "loss": 2.4622,
+      "theoretical_loss": 3.6706960622383624,
+      "tokens_seen": 940441600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003348661811991678,
+      "loss": 2.4833,
+      "theoretical_loss": 3.6694677721578377,
+      "tokens_seen": 943718400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003342751087573293,
+      "loss": 2.469,
+      "theoretical_loss": 3.66824492904894,
+      "tokens_seen": 946995200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003336840363154908,
+      "loss": 2.4464,
+      "theoretical_loss": 3.667027490033874,
+      "tokens_seen": 950272000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033309296387365234,
+      "loss": 2.4385,
+      "theoretical_loss": 3.6658154127187412,
+      "tokens_seen": 953548800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033250189143181386,
+      "loss": 2.4387,
+      "theoretical_loss": 3.664608655186437,
+      "tokens_seen": 956825600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033191081898997543,
+      "loss": 2.4551,
+      "theoretical_loss": 3.663407175989679,
+      "tokens_seen": 960102400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033131974654813695,
+      "loss": 2.484,
+      "theoretical_loss": 3.662210934144158,
+      "tokens_seen": 963379200
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003307286741062985,
+      "loss": 2.4907,
+      "theoretical_loss": 3.661019889121812,
+      "tokens_seen": 966656000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033013760166446,
+      "loss": 2.4687,
+      "theoretical_loss": 3.6598340008442234,
+      "tokens_seen": 969932800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003295465292226215,
+      "loss": 2.5139,
+      "theoretical_loss": 3.6586532296761285,
+      "tokens_seen": 973209600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032895545678078304,
+      "loss": 2.4666,
+      "theoretical_loss": 3.657477536419047,
+      "tokens_seen": 976486400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003283643843389446,
+      "loss": 2.4923,
+      "theoretical_loss": 3.656306882305022,
+      "tokens_seen": 979763200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5808769539451584,
+      "debugging/distinct-1-grams": 0.8092227708966282,
+      "debugging/distinct-2-grams": 0.969278580146789,
+      "debugging/entropy-1-grams": 5.823343222362094,
+      "debugging/entropy-2-grams": 6.624476320908173,
+      "debugging/length": 581.2857142857143,
+      "debugging/num_segments": 14,
+      "epoch": 0.35,
+      "objective/train/docs_used": 498736,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4405508041381836,
+      "objective/train/theoretical_loss": 3.6551412289904697,
+      "objective/train/tokens_used": 1003500000,
+      "theoretical_loss": 3.6551412289904697,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032777331189710613,
+      "loss": 2.4773,
+      "theoretical_loss": 3.6551412289904697,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032718223945526765,
+      "loss": 2.4446,
+      "theoretical_loss": 3.6539805385501376,
+      "tokens_seen": 986316800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003265911670134292,
+      "loss": 2.4604,
+      "theoretical_loss": 3.652824773471171,
+      "tokens_seen": 989593600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003260000945715907,
+      "loss": 2.464,
+      "theoretical_loss": 3.651673896647277,
+      "tokens_seen": 992870400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003254090221297522,
+      "loss": 2.4502,
+      "theoretical_loss": 3.6505278713729985,
+      "tokens_seen": 996147200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003248179496879138,
+      "loss": 2.4302,
+      "theoretical_loss": 3.6493866613380774,
+      "tokens_seen": 999424000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003242268772460753,
+      "loss": 2.4491,
+      "theoretical_loss": 3.648250230621924,
+      "tokens_seen": 1002700800
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032363580480423683,
+      "loss": 2.4729,
+      "theoretical_loss": 3.647118543688179,
+      "tokens_seen": 1005977600
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032304473236239835,
+      "loss": 2.4767,
+      "theoretical_loss": 3.6459915653793633,
+      "tokens_seen": 1009254400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032245365992055987,
+      "loss": 2.4782,
+      "theoretical_loss": 3.644869260911628,
+      "tokens_seen": 1012531200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003218625874787214,
+      "loss": 2.4723,
+      "theoretical_loss": 3.64375159586959,
+      "tokens_seen": 1015808000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003212715150368829,
+      "loss": 2.4532,
+      "theoretical_loss": 3.642638536201252,
+      "tokens_seen": 1019084800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003206804425950445,
+      "loss": 2.4654,
+      "theoretical_loss": 3.6415300482130135,
+      "tokens_seen": 1022361600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000320089370153206,
+      "loss": 2.4787,
+      "theoretical_loss": 3.6404260985647667,
+      "tokens_seen": 1025638400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031949829771136753,
+      "loss": 2.4936,
+      "theoretical_loss": 3.6393266542650684,
+      "tokens_seen": 1028915200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031890722526952905,
+      "loss": 2.4849,
+      "theoretical_loss": 3.638231682666401,
+      "tokens_seen": 1032192000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031831615282769057,
+      "loss": 2.455,
+      "theoretical_loss": 3.637141151460505,
+      "tokens_seen": 1035468800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003177250803858521,
+      "loss": 2.461,
+      "theoretical_loss": 3.636055028673799,
+      "tokens_seen": 1038745600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031713400794401367,
+      "loss": 2.4815,
+      "theoretical_loss": 3.634973282662864,
+      "tokens_seen": 1042022400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003165429355021752,
+      "loss": 2.462,
+      "theoretical_loss": 3.6338958821100107,
+      "tokens_seen": 1045299200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003159518630603367,
+      "loss": 2.4218,
+      "theoretical_loss": 3.63282279601892,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031536079061849823,
+      "loss": 2.435,
+      "theoretical_loss": 3.631753993710352,
+      "tokens_seen": 1051852800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031476971817665975,
+      "loss": 2.4059,
+      "theoretical_loss": 3.630689444817925,
+      "tokens_seen": 1055129600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003141786457348212,
+      "loss": 2.4132,
+      "theoretical_loss": 3.629629119283967,
+      "tokens_seen": 1058406400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003135875732929828,
+      "loss": 2.4224,
+      "theoretical_loss": 3.628572987355434,
+      "tokens_seen": 1061683200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003129965008511443,
+      "loss": 2.4123,
+      "theoretical_loss": 3.6275210195798913,
+      "tokens_seen": 1064960000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031240542840930583,
+      "loss": 2.4143,
+      "theoretical_loss": 3.626473186801564,
+      "tokens_seen": 1068236800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031181435596746736,
+      "loss": 2.4229,
+      "theoretical_loss": 3.6254294601574495,
+      "tokens_seen": 1071513600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003112232835256289,
+      "loss": 2.4552,
+      "theoretical_loss": 3.624389811073493,
+      "tokens_seen": 1074790400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003106322110837904,
+      "loss": 2.4635,
+      "theoretical_loss": 3.6233542112608257,
+      "tokens_seen": 1078067200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003100529600907888,
+      "loss": 2.4215,
+      "theoretical_loss": 3.6223226327120592,
+      "tokens_seen": 1081344000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003094618876489503,
+      "loss": 2.4313,
+      "theoretical_loss": 3.621295047697644,
+      "tokens_seen": 1084620800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003088708152071118,
+      "loss": 2.4371,
+      "theoretical_loss": 3.6202714287622833,
+      "tokens_seen": 1087897600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030827974276527334,
+      "loss": 2.4532,
+      "theoretical_loss": 3.6192517487214038,
+      "tokens_seen": 1091174400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030768867032343486,
+      "loss": 2.462,
+      "theoretical_loss": 3.6182359806576834,
+      "tokens_seen": 1094451200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003070975978815964,
+      "loss": 2.4922,
+      "theoretical_loss": 3.6172240979176333,
+      "tokens_seen": 1097728000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003065065254397579,
+      "loss": 2.5429,
+      "theoretical_loss": 3.616216074108232,
+      "tokens_seen": 1101004800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003059154529979195,
+      "loss": 2.5284,
+      "theoretical_loss": 3.6152118830936164,
+      "tokens_seen": 1104281600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000305324380556081,
+      "loss": 2.5163,
+      "theoretical_loss": 3.6142114989918195,
+      "tokens_seen": 1107558400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030473330811424246,
+      "loss": 2.539,
+      "theoretical_loss": 3.6132148961715624,
+      "tokens_seen": 1110835200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000304142235672404,
+      "loss": 2.5355,
+      "theoretical_loss": 3.6122220492490964,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003035511632305655,
+      "loss": 2.5071,
+      "theoretical_loss": 3.6112329330850894,
+      "tokens_seen": 1117388800
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000302960090788727,
+      "loss": 2.5236,
+      "theoretical_loss": 3.61024752278157,
+      "tokens_seen": 1120665600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003023690183468886,
+      "loss": 2.5306,
+      "theoretical_loss": 3.6092657936789054,
+      "tokens_seen": 1123942400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003017779459050501,
+      "loss": 2.5418,
+      "theoretical_loss": 3.6082877213528377,
+      "tokens_seen": 1127219200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030118687346321164,
+      "loss": 2.5057,
+      "theoretical_loss": 3.60731328161156,
+      "tokens_seen": 1130496000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030059580102137316,
+      "loss": 2.4979,
+      "theoretical_loss": 3.6063424504928365,
+      "tokens_seen": 1133772800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003000047285795347,
+      "loss": 2.4935,
+      "theoretical_loss": 3.60537520426117,
+      "tokens_seen": 1137049600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002994136561376962,
+      "loss": 2.4755,
+      "theoretical_loss": 3.6044115194050086,
+      "tokens_seen": 1140326400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002988225836958578,
+      "loss": 2.5173,
+      "theoretical_loss": 3.603451372633997,
+      "tokens_seen": 1143603200
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 576676,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.600228786468506,
+      "objective/train/theoretical_loss": 3.6024947408762698,
+      "objective/train/tokens_used": 1167340000,
+      "theoretical_loss": 3.6024947408762698,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002982315112540193,
+      "loss": 2.5011,
+      "theoretical_loss": 3.6024947408762698,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002976404388121808,
+      "loss": 2.4895,
+      "theoretical_loss": 3.601541601275783,
+      "tokens_seen": 1150156800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029704936637034234,
+      "loss": 2.4984,
+      "theoretical_loss": 3.6005919311896886,
+      "tokens_seen": 1153433600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029645829392850386,
+      "loss": 2.4908,
+      "theoretical_loss": 3.5996457081857454,
+      "tokens_seen": 1156710400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002958672214866654,
+      "loss": 2.5063,
+      "theoretical_loss": 3.598702910039772,
+      "tokens_seen": 1159987200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002952761490448269,
+      "loss": 2.4946,
+      "theoretical_loss": 3.597763514733133,
+      "tokens_seen": 1163264000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002946850766029885,
+      "loss": 2.48,
+      "theoretical_loss": 3.59682750045027,
+      "tokens_seen": 1166540800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029409400416115,
+      "loss": 2.46,
+      "theoretical_loss": 3.5958948455762583,
+      "tokens_seen": 1169817600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002935029317193115,
+      "loss": 2.4786,
+      "theoretical_loss": 3.594965528694412,
+      "tokens_seen": 1173094400
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029291185927747304,
+      "loss": 2.4519,
+      "theoretical_loss": 3.594039528583913,
+      "tokens_seen": 1176371200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029232078683563456,
+      "loss": 2.437,
+      "theoretical_loss": 3.5931168242174847,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002917297143937961,
+      "loss": 2.4535,
+      "theoretical_loss": 3.59219739475909,
+      "tokens_seen": 1182924800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029113864195195766,
+      "loss": 2.4221,
+      "theoretical_loss": 3.5912812195616732,
+      "tokens_seen": 1186201600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002905475695101192,
+      "loss": 2.4304,
+      "theoretical_loss": 3.590368278164926,
+      "tokens_seen": 1189478400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002899564970682807,
+      "loss": 2.4601,
+      "theoretical_loss": 3.5894585502930902,
+      "tokens_seen": 1192755200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002893654246264422,
+      "loss": 2.4571,
+      "theoretical_loss": 3.588552015852793,
+      "tokens_seen": 1196032000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028877435218460374,
+      "loss": 2.4433,
+      "theoretical_loss": 3.5876486549309097,
+      "tokens_seen": 1199308800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028818327974276526,
+      "loss": 2.4621,
+      "theoretical_loss": 3.586748447792462,
+      "tokens_seen": 1202585600
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028759220730092684,
+      "loss": 2.457,
+      "theoretical_loss": 3.5858513748785423,
+      "tokens_seen": 1205862400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028700113485908836,
+      "loss": 2.4528,
+      "theoretical_loss": 3.5849574168042704,
+      "tokens_seen": 1209139200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002864100624172499,
+      "loss": 2.4433,
+      "theoretical_loss": 3.5840665543567782,
+      "tokens_seen": 1212416000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002858189899754114,
+      "loss": 2.4581,
+      "theoretical_loss": 3.583178768493222,
+      "tokens_seen": 1215692800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002852279175335729,
+      "loss": 2.438,
+      "theoretical_loss": 3.5822940403388284,
+      "tokens_seen": 1218969600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028463684509173444,
+      "loss": 2.4308,
+      "theoretical_loss": 3.581412351184958,
+      "tokens_seen": 1222246400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028404577264989596,
+      "loss": 2.4334,
+      "theoretical_loss": 3.580533682487208,
+      "tokens_seen": 1225523200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028345470020805754,
+      "loss": 2.4651,
+      "theoretical_loss": 3.579658015863532,
+      "tokens_seen": 1228800000
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028286362776621906,
+      "loss": 2.4806,
+      "theoretical_loss": 3.5787853330923927,
+      "tokens_seen": 1232076800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002822725553243806,
+      "loss": 2.4682,
+      "theoretical_loss": 3.577915616110936,
+      "tokens_seen": 1235353600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002816814828825421,
+      "loss": 2.4349,
+      "theoretical_loss": 3.577048847013194,
+      "tokens_seen": 1238630400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002810904104407036,
+      "loss": 2.4327,
+      "theoretical_loss": 3.57618500804831,
+      "tokens_seen": 1241907200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028049933799886514,
+      "loss": 2.4281,
+      "theoretical_loss": 3.575324081618793,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002799082655570267,
+      "loss": 2.4293,
+      "theoretical_loss": 3.5744660502787875,
+      "tokens_seen": 1248460800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027931719311518824,
+      "loss": 2.434,
+      "theoretical_loss": 3.5736108967323794,
+      "tokens_seen": 1251737600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027872612067334976,
+      "loss": 2.4646,
+      "theoretical_loss": 3.5727586038319155,
+      "tokens_seen": 1255014400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002781350482315113,
+      "loss": 2.4234,
+      "theoretical_loss": 3.571909154576348,
+      "tokens_seen": 1258291200
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002775439757896728,
+      "loss": 2.4295,
+      "theoretical_loss": 3.5710625321096074,
+      "tokens_seen": 1261568000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002769529033478343,
+      "loss": 2.4476,
+      "theoretical_loss": 3.570218719718989,
+      "tokens_seen": 1264844800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002763618309059959,
+      "loss": 2.4905,
+      "theoretical_loss": 3.569377700833569,
+      "tokens_seen": 1268121600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027577075846415736,
+      "loss": 2.4667,
+      "theoretical_loss": 3.568539459022639,
+      "tokens_seen": 1271398400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002751796860223189,
+      "loss": 2.4788,
+      "theoretical_loss": 3.5677039779941584,
+      "tokens_seen": 1274675200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002745886135804804,
+      "loss": 2.4577,
+      "theoretical_loss": 3.566871241593236,
+      "tokens_seen": 1277952000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002739975411386419,
+      "loss": 2.4777,
+      "theoretical_loss": 3.5660412338006235,
+      "tokens_seen": 1281228800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027340646869680345,
+      "loss": 2.4122,
+      "theoretical_loss": 3.565213938731236,
+      "tokens_seen": 1284505600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027281539625496497,
+      "loss": 2.4321,
+      "theoretical_loss": 3.5643893406326868,
+      "tokens_seen": 1287782400
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027222432381312654,
+      "loss": 2.4425,
+      "theoretical_loss": 3.5635674238838466,
+      "tokens_seen": 1291059200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027163325137128806,
+      "loss": 2.4156,
+      "theoretical_loss": 3.5627481729934196,
+      "tokens_seen": 1294336000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002710421789294496,
+      "loss": 2.4324,
+      "theoretical_loss": 3.561931572598538,
+      "tokens_seen": 1297612800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002704511064876111,
+      "loss": 2.4463,
+      "theoretical_loss": 3.5611176074633777,
+      "tokens_seen": 1300889600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002698600340457726,
+      "loss": 2.4452,
+      "theoretical_loss": 3.5603062624777895,
+      "tokens_seen": 1304166400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026926896160393415,
+      "loss": 2.4555,
+      "theoretical_loss": 3.559497522655951,
+      "tokens_seen": 1307443200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6399427451776495,
+      "debugging/distinct-1-grams": 0.8197766779752171,
+      "debugging/distinct-2-grams": 0.9705234588832752,
+      "debugging/entropy-1-grams": 5.882641793405599,
+      "debugging/entropy-2-grams": 6.547451268934809,
+      "debugging/length": 584.7857142857143,
+      "debugging/num_segments": 14,
+      "epoch": 0.47,
+      "objective/train/docs_used": 655957,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.211557388305664,
+      "objective/train/theoretical_loss": 3.5586913731350327,
+      "objective/train/tokens_used": 1331180000,
+      "theoretical_loss": 3.5586913731350327,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002686778891620957,
+      "loss": 2.413,
+      "theoretical_loss": 3.5586913731350327,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026808681672025724,
+      "loss": 2.4279,
+      "theoretical_loss": 3.557887799173889,
+      "tokens_seen": 1313996800
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026749574427841876,
+      "loss": 2.4284,
+      "theoretical_loss": 3.557086786151754,
+      "tokens_seen": 1317273600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002669046718365803,
+      "loss": 2.4231,
+      "theoretical_loss": 3.5562883195669697,
+      "tokens_seen": 1320550400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002663135993947418,
+      "loss": 2.4112,
+      "theoretical_loss": 3.555492385035719,
+      "tokens_seen": 1323827200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002657225269529033,
+      "loss": 2.4624,
+      "theoretical_loss": 3.5546989682907784,
+      "tokens_seen": 1327104000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002651314545110649,
+      "loss": 2.4257,
+      "theoretical_loss": 3.5539080551802895,
+      "tokens_seen": 1330380800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002645403820692264,
+      "loss": 2.4078,
+      "theoretical_loss": 3.553119631666546,
+      "tokens_seen": 1333657600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026394930962738794,
+      "loss": 2.4365,
+      "theoretical_loss": 3.5523336838247914,
+      "tokens_seen": 1336934400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026335823718554946,
+      "loss": 2.428,
+      "theoretical_loss": 3.55155019784204,
+      "tokens_seen": 1340211200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000262767164743711,
+      "loss": 2.4614,
+      "theoretical_loss": 3.5507691600159053,
+      "tokens_seen": 1343488000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002621760923018725,
+      "loss": 2.4644,
+      "theoretical_loss": 3.5499905567534515,
+      "tokens_seen": 1346764800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.000261585019860034,
+      "loss": 2.4448,
+      "theoretical_loss": 3.549214374570052,
+      "tokens_seen": 1350041600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002609939474181956,
+      "loss": 2.4452,
+      "theoretical_loss": 3.5484406000882665,
+      "tokens_seen": 1353318400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002604028749763571,
+      "loss": 2.462,
+      "theoretical_loss": 3.5476692200367346,
+      "tokens_seen": 1356595200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025981180253451864,
+      "loss": 2.4537,
+      "theoretical_loss": 3.546900221249076,
+      "tokens_seen": 1359872000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025922073009268016,
+      "loss": 2.4445,
+      "theoretical_loss": 3.5461335906628157,
+      "tokens_seen": 1363148800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002586296576508417,
+      "loss": 2.4372,
+      "theoretical_loss": 3.54536931531831,
+      "tokens_seen": 1366425600
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002580385852090032,
+      "loss": 2.4444,
+      "theoretical_loss": 3.5446073823576985,
+      "tokens_seen": 1369702400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002574475127671648,
+      "loss": 2.4888,
+      "theoretical_loss": 3.543847779023859,
+      "tokens_seen": 1372979200
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002568564403253263,
+      "loss": 2.4657,
+      "theoretical_loss": 3.543090492659384,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002562653678834878,
+      "loss": 2.4599,
+      "theoretical_loss": 3.542335510705562,
+      "tokens_seen": 1379532800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025567429544164934,
+      "loss": 2.4652,
+      "theoretical_loss": 3.541582820701378,
+      "tokens_seen": 1382809600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025508322299981086,
+      "loss": 2.4831,
+      "theoretical_loss": 3.5408324102825253,
+      "tokens_seen": 1386086400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002544921505579724,
+      "loss": 2.4716,
+      "theoretical_loss": 3.5400842671804265,
+      "tokens_seen": 1389363200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025390107811613396,
+      "loss": 2.4304,
+      "theoretical_loss": 3.5393383792212676,
+      "tokens_seen": 1392640000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002533100056742955,
+      "loss": 2.4272,
+      "theoretical_loss": 3.5385947343250486,
+      "tokens_seen": 1395916800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000252718933232457,
+      "loss": 2.437,
+      "theoretical_loss": 3.53785332050464,
+      "tokens_seen": 1399193600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002521278607906185,
+      "loss": 2.4381,
+      "theoretical_loss": 3.537114125864853,
+      "tokens_seen": 1402470400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025153678834878004,
+      "loss": 2.4566,
+      "theoretical_loss": 3.536377138601523,
+      "tokens_seen": 1405747200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025094571590694156,
+      "loss": 2.4318,
+      "theoretical_loss": 3.5356423470006018,
+      "tokens_seen": 1409024000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025035464346510314,
+      "loss": 2.4233,
+      "theoretical_loss": 3.5349097394372615,
+      "tokens_seen": 1412300800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002497635710232646,
+      "loss": 2.389,
+      "theoretical_loss": 3.5341793043750114,
+      "tokens_seen": 1415577600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002491724985814262,
+      "loss": 2.3994,
+      "theoretical_loss": 3.533451030364823,
+      "tokens_seen": 1418854400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002485814261395877,
+      "loss": 2.399,
+      "theoretical_loss": 3.5327249060442676,
+      "tokens_seen": 1422131200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002479903536977492,
+      "loss": 2.4178,
+      "theoretical_loss": 3.5320009201366616,
+      "tokens_seen": 1425408000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024739928125591074,
+      "loss": 2.4211,
+      "theoretical_loss": 3.531279061450226,
+      "tokens_seen": 1428684800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024680820881407226,
+      "loss": 2.3951,
+      "theoretical_loss": 3.5305593188772533,
+      "tokens_seen": 1431961600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002462171363722338,
+      "loss": 2.4152,
+      "theoretical_loss": 3.5298416813932865,
+      "tokens_seen": 1435238400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002456260639303953,
+      "loss": 2.4106,
+      "theoretical_loss": 3.529126138056305,
+      "tokens_seen": 1438515200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002450349914885568,
+      "loss": 2.4094,
+      "theoretical_loss": 3.528412678005922,
+      "tokens_seen": 1441792000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024444391904671834,
+      "loss": 2.3699,
+      "theoretical_loss": 3.527701290462595,
+      "tokens_seen": 1445068800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002438528466048799,
+      "loss": 2.364,
+      "theoretical_loss": 3.5269919647268377,
+      "tokens_seen": 1448345600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024326177416304144,
+      "loss": 2.4052,
+      "theoretical_loss": 3.526284690178448,
+      "tokens_seen": 1451622400
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024267070172120296,
+      "loss": 2.3649,
+      "theoretical_loss": 3.5255794562757443,
+      "tokens_seen": 1454899200
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024207962927936448,
+      "loss": 2.3964,
+      "theoretical_loss": 3.524876252554808,
+      "tokens_seen": 1458176000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024148855683752603,
+      "loss": 2.3963,
+      "theoretical_loss": 3.524175068628737,
+      "tokens_seen": 1461452800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024089748439568755,
+      "loss": 2.3774,
+      "theoretical_loss": 3.5234758941869075,
+      "tokens_seen": 1464729600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024030641195384907,
+      "loss": 2.3943,
+      "theoretical_loss": 3.522778718994246,
+      "tokens_seen": 1468006400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002397153395120106,
+      "loss": 2.3874,
+      "theoretical_loss": 3.5220835328905062,
+      "tokens_seen": 1471283200
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 736371,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.6813464164733887,
+      "objective/train/theoretical_loss": 3.521390325789559,
+      "objective/train/tokens_used": 1495020000,
+      "theoretical_loss": 3.521390325789559,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002391242670701721,
+      "loss": 2.4096,
+      "theoretical_loss": 3.521390325789559,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023853319462833363,
+      "loss": 2.4186,
+      "theoretical_loss": 3.520699087678688,
+      "tokens_seen": 1477836800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023794212218649518,
+      "loss": 2.4153,
+      "theoretical_loss": 3.5200098086178926,
+      "tokens_seen": 1481113600
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002373510497446567,
+      "loss": 2.4079,
+      "theoretical_loss": 3.5193224787392015,
+      "tokens_seen": 1484390400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023675997730281822,
+      "loss": 2.4285,
+      "theoretical_loss": 3.518637088245993,
+      "tokens_seen": 1487667200
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023616890486097977,
+      "loss": 2.401,
+      "theoretical_loss": 3.5179536274123233,
+      "tokens_seen": 1490944000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002355778324191413,
+      "loss": 2.3969,
+      "theoretical_loss": 3.5172720865822633,
+      "tokens_seen": 1494220800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002349867599773028,
+      "loss": 2.4153,
+      "theoretical_loss": 3.5165924561692394,
+      "tokens_seen": 1497497600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023439568753546436,
+      "loss": 2.4198,
+      "theoretical_loss": 3.5159147266553887,
+      "tokens_seen": 1500774400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023380461509362588,
+      "loss": 2.424,
+      "theoretical_loss": 3.515238888590916,
+      "tokens_seen": 1504051200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002332135426517874,
+      "loss": 2.4336,
+      "theoretical_loss": 3.51456493259346,
+      "tokens_seen": 1507328000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023262247020994895,
+      "loss": 2.4143,
+      "theoretical_loss": 3.5138928493474664,
+      "tokens_seen": 1510604800
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023203139776811047,
+      "loss": 2.3933,
+      "theoretical_loss": 3.513222629603569,
+      "tokens_seen": 1513881600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.000231440325326272,
+      "loss": 2.3996,
+      "theoretical_loss": 3.512554264177979,
+      "tokens_seen": 1517158400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002308492528844335,
+      "loss": 2.4192,
+      "theoretical_loss": 3.511887743951874,
+      "tokens_seen": 1520435200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023025818044259506,
+      "loss": 2.3924,
+      "theoretical_loss": 3.511223059870809,
+      "tokens_seen": 1523712000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022966710800075658,
+      "loss": 2.4092,
+      "theoretical_loss": 3.5105602029441143,
+      "tokens_seen": 1526988800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002290760355589181,
+      "loss": 2.4007,
+      "theoretical_loss": 3.5098991642443185,
+      "tokens_seen": 1530265600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022848496311707965,
+      "loss": 2.3678,
+      "theoretical_loss": 3.5092399349065655,
+      "tokens_seen": 1533542400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022789389067524117,
+      "loss": 2.4004,
+      "theoretical_loss": 3.5085825061280445,
+      "tokens_seen": 1536819200
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002273028182334027,
+      "loss": 2.4112,
+      "theoretical_loss": 3.507926869167425,
+      "tokens_seen": 1540096000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022671174579156424,
+      "loss": 2.4276,
+      "theoretical_loss": 3.507273015344296,
+      "tokens_seen": 1543372800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022612067334972576,
+      "loss": 2.4327,
+      "theoretical_loss": 3.5066209360386154,
+      "tokens_seen": 1546649600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022552960090788728,
+      "loss": 2.417,
+      "theoretical_loss": 3.5059706226901595,
+      "tokens_seen": 1549926400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002249385284660488,
+      "loss": 2.4228,
+      "theoretical_loss": 3.505322066797988,
+      "tokens_seen": 1553203200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022434745602421032,
+      "loss": 2.464,
+      "theoretical_loss": 3.504675259919904,
+      "tokens_seen": 1556480000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022376820503120862,
+      "loss": 2.4151,
+      "theoretical_loss": 3.5040301936719302,
+      "tokens_seen": 1559756800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022318895403820695,
+      "loss": 2.4042,
+      "theoretical_loss": 3.5033868597277813,
+      "tokens_seen": 1563033600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022259788159636847,
+      "loss": 2.4284,
+      "theoretical_loss": 3.502745249818351,
+      "tokens_seen": 1566310400
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022200680915453,
+      "loss": 2.4027,
+      "theoretical_loss": 3.5021053557311985,
+      "tokens_seen": 1569587200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022141573671269154,
+      "loss": 2.3931,
+      "theoretical_loss": 3.501467169310044,
+      "tokens_seen": 1572864000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022082466427085303,
+      "loss": 2.4082,
+      "theoretical_loss": 3.500830682454268,
+      "tokens_seen": 1576140800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022023359182901455,
+      "loss": 2.4037,
+      "theoretical_loss": 3.5001958871184167,
+      "tokens_seen": 1579417600
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002196425193871761,
+      "loss": 2.3834,
+      "theoretical_loss": 3.4995627753117127,
+      "tokens_seen": 1582694400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021905144694533762,
+      "loss": 2.4117,
+      "theoretical_loss": 3.498931339097571,
+      "tokens_seen": 1585971200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021846037450349914,
+      "loss": 2.4118,
+      "theoretical_loss": 3.4983015705931226,
+      "tokens_seen": 1589248000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021786930206166066,
+      "loss": 2.3917,
+      "theoretical_loss": 3.4976734619687377,
+      "tokens_seen": 1592524800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002172782296198222,
+      "loss": 2.385,
+      "theoretical_loss": 3.497047005447558,
+      "tokens_seen": 1595801600
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021668715717798373,
+      "loss": 2.3604,
+      "theoretical_loss": 3.4964221933050377,
+      "tokens_seen": 1599078400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021609608473614525,
+      "loss": 2.3923,
+      "theoretical_loss": 3.495799017868479,
+      "tokens_seen": 1602355200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002155050122943068,
+      "loss": 2.3634,
+      "theoretical_loss": 3.495177471516584,
+      "tokens_seen": 1605632000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021491393985246832,
+      "loss": 2.4133,
+      "theoretical_loss": 3.4945575466790024,
+      "tokens_seen": 1608908800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021432286741062984,
+      "loss": 2.3991,
+      "theoretical_loss": 3.4939392358358923,
+      "tokens_seen": 1612185600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002137317949687914,
+      "loss": 2.3925,
+      "theoretical_loss": 3.493322531517478,
+      "tokens_seen": 1615462400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002131407225269529,
+      "loss": 2.3672,
+      "theoretical_loss": 3.4927074263036157,
+      "tokens_seen": 1618739200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021254965008511443,
+      "loss": 2.3873,
+      "theoretical_loss": 3.492093912823367,
+      "tokens_seen": 1622016000
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021195857764327598,
+      "loss": 2.382,
+      "theoretical_loss": 3.4914819837545705,
+      "tokens_seen": 1625292800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002113675052014375,
+      "loss": 2.3346,
+      "theoretical_loss": 3.490871631823426,
+      "tokens_seen": 1628569600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021077643275959902,
+      "loss": 2.3335,
+      "theoretical_loss": 3.490262849804073,
+      "tokens_seen": 1631846400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021018536031776057,
+      "loss": 2.3264,
+      "theoretical_loss": 3.4896556305181834,
+      "tokens_seen": 1635123200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6576681291722434,
+      "debugging/distinct-1-grams": 0.7857624523046014,
+      "debugging/distinct-2-grams": 0.9298378001713203,
+      "debugging/entropy-1-grams": 5.805497256659562,
+      "debugging/entropy-2-grams": 6.5496782401831934,
+      "debugging/length": 685.0,
+      "debugging/num_segments": 12,
+      "epoch": 0.59,
+      "objective/train/docs_used": 820116,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4782586097717285,
+      "objective/train/theoretical_loss": 3.4890499668345516,
+      "objective/train/tokens_used": 1658860000,
+      "theoretical_loss": 3.4890499668345516,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002095942878759221,
+      "loss": 2.3316,
+      "theoretical_loss": 3.4890499668345516,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002090032154340836,
+      "loss": 2.3412,
+      "theoretical_loss": 3.4884458516686947,
+      "tokens_seen": 1641676800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020841214299224516,
+      "loss": 2.3587,
+      "theoretical_loss": 3.487843277982451,
+      "tokens_seen": 1644953600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020782107055040668,
+      "loss": 2.3609,
+      "theoretical_loss": 3.4872422387835877,
+      "tokens_seen": 1648230400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002072299981085682,
+      "loss": 2.3427,
+      "theoretical_loss": 3.4866427271254077,
+      "tokens_seen": 1651507200
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020663892566672972,
+      "loss": 2.3388,
+      "theoretical_loss": 3.4860447361063667,
+      "tokens_seen": 1654784000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020604785322489124,
+      "loss": 2.3563,
+      "theoretical_loss": 3.48544825886969,
+      "tokens_seen": 1658060800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020545678078305276,
+      "loss": 2.3842,
+      "theoretical_loss": 3.4848532886029924,
+      "tokens_seen": 1661337600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020486570834121428,
+      "loss": 2.363,
+      "theoretical_loss": 3.4842598185379066,
+      "tokens_seen": 1664614400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020427463589937583,
+      "loss": 2.3799,
+      "theoretical_loss": 3.4836678419497105,
+      "tokens_seen": 1667891200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020368356345753735,
+      "loss": 2.3802,
+      "theoretical_loss": 3.483077352156964,
+      "tokens_seen": 1671168000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020309249101569887,
+      "loss": 2.3742,
+      "theoretical_loss": 3.4824883425211413,
+      "tokens_seen": 1674444800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020250141857386042,
+      "loss": 2.3967,
+      "theoretical_loss": 3.4819008064462764,
+      "tokens_seen": 1677721600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020191034613202194,
+      "loss": 2.4032,
+      "theoretical_loss": 3.481314737378605,
+      "tokens_seen": 1680998400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020131927369018346,
+      "loss": 2.4034,
+      "theoretical_loss": 3.4807301288062145,
+      "tokens_seen": 1684275200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.000200728201248345,
+      "loss": 2.4327,
+      "theoretical_loss": 3.4801469742586955,
+      "tokens_seen": 1687552000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020013712880650653,
+      "loss": 2.4095,
+      "theoretical_loss": 3.4795652673067954,
+      "tokens_seen": 1690828800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019954605636466805,
+      "loss": 2.4408,
+      "theoretical_loss": 3.478985001562082,
+      "tokens_seen": 1694105600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001989549839228296,
+      "loss": 2.4046,
+      "theoretical_loss": 3.4784061706766,
+      "tokens_seen": 1697382400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019836391148099112,
+      "loss": 2.4231,
+      "theoretical_loss": 3.477828768342543,
+      "tokens_seen": 1700659200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019777283903915264,
+      "loss": 2.4127,
+      "theoretical_loss": 3.477252788291917,
+      "tokens_seen": 1703936000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001971817665973142,
+      "loss": 2.3706,
+      "theoretical_loss": 3.4766782242962186,
+      "tokens_seen": 1707212800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001965906941554757,
+      "loss": 2.4046,
+      "theoretical_loss": 3.476105070166107,
+      "tokens_seen": 1710489600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019599962171363723,
+      "loss": 2.4168,
+      "theoretical_loss": 3.475533319751085,
+      "tokens_seen": 1713766400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019540854927179878,
+      "loss": 2.3621,
+      "theoretical_loss": 3.4749629669391813,
+      "tokens_seen": 1717043200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001948174768299603,
+      "loss": 2.3988,
+      "theoretical_loss": 3.474394005656636,
+      "tokens_seen": 1720320000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019422640438812182,
+      "loss": 2.3692,
+      "theoretical_loss": 3.473826429867589,
+      "tokens_seen": 1723596800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019363533194628334,
+      "loss": 2.3768,
+      "theoretical_loss": 3.4732602335737734,
+      "tokens_seen": 1726873600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001930442595044449,
+      "loss": 2.3593,
+      "theoretical_loss": 3.47269541081421,
+      "tokens_seen": 1730150400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019245318706260638,
+      "loss": 2.3602,
+      "theoretical_loss": 3.4721319556649046,
+      "tokens_seen": 1733427200
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001918621146207679,
+      "loss": 2.3361,
+      "theoretical_loss": 3.4715698622385487,
+      "tokens_seen": 1736704000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019127104217892945,
+      "loss": 2.3859,
+      "theoretical_loss": 3.471009124684226,
+      "tokens_seen": 1739980800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019067996973709097,
+      "loss": 2.3795,
+      "theoretical_loss": 3.470449737187118,
+      "tokens_seen": 1743257600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001900888972952525,
+      "loss": 2.356,
+      "theoretical_loss": 3.4698916939682123,
+      "tokens_seen": 1746534400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018949782485341404,
+      "loss": 2.3867,
+      "theoretical_loss": 3.4693349892840186,
+      "tokens_seen": 1749811200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018890675241157556,
+      "loss": 2.3647,
+      "theoretical_loss": 3.4687796174262826,
+      "tokens_seen": 1753088000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018831567996973708,
+      "loss": 2.3858,
+      "theoretical_loss": 3.468225572721704,
+      "tokens_seen": 1756364800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018772460752789863,
+      "loss": 2.3647,
+      "theoretical_loss": 3.4676728495316587,
+      "tokens_seen": 1759641600
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018713353508606015,
+      "loss": 2.3535,
+      "theoretical_loss": 3.467121442251922,
+      "tokens_seen": 1762918400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018654246264422167,
+      "loss": 2.3634,
+      "theoretical_loss": 3.4665713453123965,
+      "tokens_seen": 1766195200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018595139020238322,
+      "loss": 2.3299,
+      "theoretical_loss": 3.46602255317684,
+      "tokens_seen": 1769472000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018536031776054474,
+      "loss": 2.3638,
+      "theoretical_loss": 3.465475060342598,
+      "tokens_seen": 1772748800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018476924531870626,
+      "loss": 2.3668,
+      "theoretical_loss": 3.4649288613403386,
+      "tokens_seen": 1776025600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001841781728768678,
+      "loss": 2.3603,
+      "theoretical_loss": 3.4643839507337892,
+      "tokens_seen": 1779302400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018358710043502933,
+      "loss": 2.3772,
+      "theoretical_loss": 3.4638403231194763,
+      "tokens_seen": 1782579200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018299602799319085,
+      "loss": 2.4006,
+      "theoretical_loss": 3.463297973126469,
+      "tokens_seen": 1785856000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018240495555135237,
+      "loss": 2.3972,
+      "theoretical_loss": 3.462756895416121,
+      "tokens_seen": 1789132800
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018181388310951392,
+      "loss": 2.3698,
+      "theoretical_loss": 3.4622170846818205,
+      "tokens_seen": 1792409600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018122281066767544,
+      "loss": 2.3579,
+      "theoretical_loss": 3.4616785356487387,
+      "tokens_seen": 1795686400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018063173822583696,
+      "loss": 2.3544,
+      "theoretical_loss": 3.461141243073582,
+      "tokens_seen": 1798963200
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 901337,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 1.980056643486023,
+      "objective/train/theoretical_loss": 3.4606052017443467,
+      "objective/train/tokens_used": 1822700000,
+      "theoretical_loss": 3.4606052017443467,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001800406657839985,
+      "loss": 2.3462,
+      "theoretical_loss": 3.4606052017443467,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017944959334216003,
+      "loss": 2.3869,
+      "theoretical_loss": 3.4600704064800745,
+      "tokens_seen": 1805516800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017885852090032155,
+      "loss": 2.3593,
+      "theoretical_loss": 3.459536852130613,
+      "tokens_seen": 1808793600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001782674484584831,
+      "loss": 2.3634,
+      "theoretical_loss": 3.4590045335763775,
+      "tokens_seen": 1812070400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001776763760166446,
+      "loss": 2.3412,
+      "theoretical_loss": 3.4584734457281128,
+      "tokens_seen": 1815347200
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001770853035748061,
+      "loss": 2.3558,
+      "theoretical_loss": 3.457943583526662,
+      "tokens_seen": 1818624000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017649423113296766,
+      "loss": 2.3537,
+      "theoretical_loss": 3.45741494194273,
+      "tokens_seen": 1821900800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017590315869112918,
+      "loss": 2.3762,
+      "theoretical_loss": 3.4568875159766606,
+      "tokens_seen": 1825177600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001753120862492907,
+      "loss": 2.4013,
+      "theoretical_loss": 3.4563613006582017,
+      "tokens_seen": 1828454400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017472101380745225,
+      "loss": 2.3629,
+      "theoretical_loss": 3.4558362910462863,
+      "tokens_seen": 1831731200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017412994136561377,
+      "loss": 2.3589,
+      "theoretical_loss": 3.4553124822288055,
+      "tokens_seen": 1835008000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001735388689237753,
+      "loss": 2.3649,
+      "theoretical_loss": 3.4547898693223873,
+      "tokens_seen": 1838284800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017294779648193684,
+      "loss": 2.3486,
+      "theoretical_loss": 3.4542684474721805,
+      "tokens_seen": 1841561600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017235672404009836,
+      "loss": 2.3414,
+      "theoretical_loss": 3.453748211851636,
+      "tokens_seen": 1844838400
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017176565159825988,
+      "loss": 2.3242,
+      "theoretical_loss": 3.453229157662291,
+      "tokens_seen": 1848115200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017117457915642143,
+      "loss": 2.3426,
+      "theoretical_loss": 3.452711280133558,
+      "tokens_seen": 1851392000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017058350671458295,
+      "loss": 2.3607,
+      "theoretical_loss": 3.4521945745225118,
+      "tokens_seen": 1854668800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016999243427274447,
+      "loss": 2.332,
+      "theoretical_loss": 3.451679036113682,
+      "tokens_seen": 1857945600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.000169401361830906,
+      "loss": 2.3366,
+      "theoretical_loss": 3.451164660218848,
+      "tokens_seen": 1861222400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016881028938906754,
+      "loss": 2.361,
+      "theoretical_loss": 3.4506514421768273,
+      "tokens_seen": 1864499200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016821921694722906,
+      "loss": 2.3613,
+      "theoretical_loss": 3.45013937735328,
+      "tokens_seen": 1867776000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016762814450539058,
+      "loss": 2.3651,
+      "theoretical_loss": 3.4496284611405024,
+      "tokens_seen": 1871052800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016703707206355213,
+      "loss": 2.3338,
+      "theoretical_loss": 3.4491186889572303,
+      "tokens_seen": 1874329600
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016644599962171365,
+      "loss": 2.3269,
+      "theoretical_loss": 3.4486100562484396,
+      "tokens_seen": 1877606400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016585492717987517,
+      "loss": 2.3004,
+      "theoretical_loss": 3.448102558485152,
+      "tokens_seen": 1880883200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016526385473803672,
+      "loss": 2.3284,
+      "theoretical_loss": 3.447596191164239,
+      "tokens_seen": 1884160000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016467278229619824,
+      "loss": 2.3293,
+      "theoretical_loss": 3.447090949808233,
+      "tokens_seen": 1887436800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016408170985435976,
+      "loss": 2.3144,
+      "theoretical_loss": 3.446586829965132,
+      "tokens_seen": 1890713600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001634906374125213,
+      "loss": 2.3558,
+      "theoretical_loss": 3.4460838272082173,
+      "tokens_seen": 1893990400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001628995649706828,
+      "loss": 2.3579,
+      "theoretical_loss": 3.4455819371358594,
+      "tokens_seen": 1897267200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016230849252884432,
+      "loss": 2.3538,
+      "theoretical_loss": 3.445081155371338,
+      "tokens_seen": 1900544000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016171742008700587,
+      "loss": 2.3435,
+      "theoretical_loss": 3.4445814775626564,
+      "tokens_seen": 1903820800
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001611263476451674,
+      "loss": 2.3301,
+      "theoretical_loss": 3.4440828993823605,
+      "tokens_seen": 1907097600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001605352752033289,
+      "loss": 2.3699,
+      "theoretical_loss": 3.4435854165273567,
+      "tokens_seen": 1910374400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015994420276149046,
+      "loss": 2.3923,
+      "theoretical_loss": 3.443089024718735,
+      "tokens_seen": 1913651200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015935313031965198,
+      "loss": 2.3705,
+      "theoretical_loss": 3.4425937197015912,
+      "tokens_seen": 1916928000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001587620578778135,
+      "loss": 2.3477,
+      "theoretical_loss": 3.4420994972448513,
+      "tokens_seen": 1920204800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015817098543597502,
+      "loss": 2.3717,
+      "theoretical_loss": 3.4416063531410988,
+      "tokens_seen": 1923481600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015757991299413657,
+      "loss": 2.3713,
+      "theoretical_loss": 3.4411142832064003,
+      "tokens_seen": 1926758400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001569888405522981,
+      "loss": 2.3732,
+      "theoretical_loss": 3.4406232832801376,
+      "tokens_seen": 1930035200
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001563977681104596,
+      "loss": 2.3423,
+      "theoretical_loss": 3.440133349224835,
+      "tokens_seen": 1933312000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015580669566862116,
+      "loss": 2.339,
+      "theoretical_loss": 3.439644476925995,
+      "tokens_seen": 1936588800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015521562322678268,
+      "loss": 2.3397,
+      "theoretical_loss": 3.4391566622919303,
+      "tokens_seen": 1939865600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001546245507849442,
+      "loss": 2.3574,
+      "theoretical_loss": 3.438669901253598,
+      "tokens_seen": 1943142400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015403347834310575,
+      "loss": 2.3522,
+      "theoretical_loss": 3.4381841897644385,
+      "tokens_seen": 1946419200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015344240590126727,
+      "loss": 2.3096,
+      "theoretical_loss": 3.4376995238002137,
+      "tokens_seen": 1949696000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001528513334594288,
+      "loss": 2.3218,
+      "theoretical_loss": 3.4372158993588444,
+      "tokens_seen": 1952972800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001522720824664271,
+      "loss": 2.3679,
+      "theoretical_loss": 3.436733312460253,
+      "tokens_seen": 1956249600
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001516810100245886,
+      "loss": 2.3815,
+      "theoretical_loss": 3.436251759146206,
+      "tokens_seen": 1959526400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015108993758275013,
+      "loss": 2.3897,
+      "theoretical_loss": 3.435771235480157,
+      "tokens_seen": 1962803200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6190456803976605,
+      "debugging/distinct-1-grams": 0.810426091495274,
+      "debugging/distinct-2-grams": 0.9774878499008586,
+      "debugging/entropy-1-grams": 5.696312135904344,
+      "debugging/entropy-2-grams": 6.389064417845406,
+      "debugging/length": 634.8181818181819,
+      "debugging/num_segments": 11,
+      "epoch": 0.7,
+      "objective/train/docs_used": 981242,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4205574989318848,
+      "objective/train/theoretical_loss": 3.435291737547092,
+      "objective/train/tokens_used": 1986540000,
+      "theoretical_loss": 3.435291737547092,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015051068658974846,
+      "loss": 2.3887,
+      "theoretical_loss": 3.435291737547092,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014991961414790998,
+      "loss": 2.3826,
+      "theoretical_loss": 3.434813261453378,
+      "tokens_seen": 1969356800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001493285417060715,
+      "loss": 2.3929,
+      "theoretical_loss": 3.434335803326607,
+      "tokens_seen": 1972633600
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014873746926423304,
+      "loss": 2.4085,
+      "theoretical_loss": 3.433859359315448,
+      "tokens_seen": 1975910400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014814639682239457,
+      "loss": 2.3988,
+      "theoretical_loss": 3.433383925589498,
+      "tokens_seen": 1979187200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014755532438055609,
+      "loss": 2.3973,
+      "theoretical_loss": 3.432909498339132,
+      "tokens_seen": 1982464000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014696425193871763,
+      "loss": 2.3857,
+      "theoretical_loss": 3.432436073775358,
+      "tokens_seen": 1985740800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014637317949687915,
+      "loss": 2.3655,
+      "theoretical_loss": 3.4319636481296687,
+      "tokens_seen": 1989017600
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014578210705504068,
+      "loss": 2.3756,
+      "theoretical_loss": 3.4314922176539002,
+      "tokens_seen": 1992294400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014519103461320222,
+      "loss": 2.3543,
+      "theoretical_loss": 3.431021778620088,
+      "tokens_seen": 1995571200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014459996217136372,
+      "loss": 2.3713,
+      "theoretical_loss": 3.4305523273203233,
+      "tokens_seen": 1998848000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014400888972952524,
+      "loss": 2.3627,
+      "theoretical_loss": 3.430083860066617,
+      "tokens_seen": 2002124800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014341781728768676,
+      "loss": 2.3517,
+      "theoretical_loss": 3.429616373190753,
+      "tokens_seen": 2005401600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001428267448458483,
+      "loss": 2.3738,
+      "theoretical_loss": 3.4291498630441586,
+      "tokens_seen": 2008678400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014223567240400983,
+      "loss": 2.3361,
+      "theoretical_loss": 3.4286843259977613,
+      "tokens_seen": 2011955200
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014164459996217135,
+      "loss": 2.3546,
+      "theoretical_loss": 3.4282197584418546,
+      "tokens_seen": 2015232000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001410535275203329,
+      "loss": 2.3561,
+      "theoretical_loss": 3.427756156785965,
+      "tokens_seen": 2018508800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014046245507849442,
+      "loss": 2.3122,
+      "theoretical_loss": 3.427293517458717,
+      "tokens_seen": 2021785600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013987138263665594,
+      "loss": 2.3601,
+      "theoretical_loss": 3.4268318369076995,
+      "tokens_seen": 2025062400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013928031019481749,
+      "loss": 2.3506,
+      "theoretical_loss": 3.4263711115993383,
+      "tokens_seen": 2028339200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.000138689237752979,
+      "loss": 2.3668,
+      "theoretical_loss": 3.425911338018763,
+      "tokens_seen": 2031616000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013809816531114053,
+      "loss": 2.3032,
+      "theoretical_loss": 3.425452512669678,
+      "tokens_seen": 2034892800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013750709286930208,
+      "loss": 2.3432,
+      "theoretical_loss": 3.4249946320742364,
+      "tokens_seen": 2038169600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001369160204274636,
+      "loss": 2.3589,
+      "theoretical_loss": 3.4245376927729128,
+      "tokens_seen": 2041446400
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013632494798562512,
+      "loss": 2.315,
+      "theoretical_loss": 3.424081691324375,
+      "tokens_seen": 2044723200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013573387554378666,
+      "loss": 2.3221,
+      "theoretical_loss": 3.4236266243053626,
+      "tokens_seen": 2048000000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013514280310194819,
+      "loss": 2.3323,
+      "theoretical_loss": 3.4231724883105628,
+      "tokens_seen": 2051276800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001345517306601097,
+      "loss": 2.3435,
+      "theoretical_loss": 3.422719279952486,
+      "tokens_seen": 2054553600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013396065821827125,
+      "loss": 2.319,
+      "theoretical_loss": 3.422266995861345,
+      "tokens_seen": 2057830400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013336958577643277,
+      "loss": 2.3608,
+      "theoretical_loss": 3.421815632684937,
+      "tokens_seen": 2061107200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001327785133345943,
+      "loss": 2.3364,
+      "theoretical_loss": 3.4213651870885196,
+      "tokens_seen": 2064384000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013218744089275582,
+      "loss": 2.3433,
+      "theoretical_loss": 3.4209156557546954,
+      "tokens_seen": 2067660800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013160818989975412,
+      "loss": 2.3064,
+      "theoretical_loss": 3.420467035383294,
+      "tokens_seen": 2070937600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013101711745791564,
+      "loss": 2.3062,
+      "theoretical_loss": 3.4200193226912554,
+      "tokens_seen": 2074214400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013042604501607716,
+      "loss": 2.3204,
+      "theoretical_loss": 3.419572514412513,
+      "tokens_seen": 2077491200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001298349725742387,
+      "loss": 2.3435,
+      "theoretical_loss": 3.4191266072978816,
+      "tokens_seen": 2080768000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012924390013240023,
+      "loss": 2.34,
+      "theoretical_loss": 3.418681598114942,
+      "tokens_seen": 2084044800
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012865282769056175,
+      "loss": 2.3336,
+      "theoretical_loss": 3.4182374836479266,
+      "tokens_seen": 2087321600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001280617552487233,
+      "loss": 2.367,
+      "theoretical_loss": 3.4177942606976126,
+      "tokens_seen": 2090598400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012747068280688482,
+      "loss": 2.4024,
+      "theoretical_loss": 3.4173519260812064,
+      "tokens_seen": 2093875200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012687961036504634,
+      "loss": 2.3796,
+      "theoretical_loss": 3.4169104766322356,
+      "tokens_seen": 2097152000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012628853792320788,
+      "loss": 2.3606,
+      "theoretical_loss": 3.4164699092004405,
+      "tokens_seen": 2100428800
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001256974654813694,
+      "loss": 2.3834,
+      "theoretical_loss": 3.416030220651664,
+      "tokens_seen": 2103705600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012510639303953093,
+      "loss": 2.4119,
+      "theoretical_loss": 3.4155914078677467,
+      "tokens_seen": 2106982400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012451532059769245,
+      "loss": 2.387,
+      "theoretical_loss": 3.4151534677464195,
+      "tokens_seen": 2110259200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.000123924248155854,
+      "loss": 2.3655,
+      "theoretical_loss": 3.414716397201196,
+      "tokens_seen": 2113536000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012333317571401552,
+      "loss": 2.3614,
+      "theoretical_loss": 3.4142801931612725,
+      "tokens_seen": 2116812800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012274210327217704,
+      "loss": 2.3275,
+      "theoretical_loss": 3.41384485257142,
+      "tokens_seen": 2120089600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012215103083033858,
+      "loss": 2.3087,
+      "theoretical_loss": 3.4134103723918834,
+      "tokens_seen": 2123366400
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012155995838850009,
+      "loss": 2.3282,
+      "theoretical_loss": 3.412976749598278,
+      "tokens_seen": 2126643200
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/docs_used": 1063161,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.2789740562438965,
+      "objective/train/theoretical_loss": 3.412543981181491,
+      "objective/train/tokens_used": 2150380000,
+      "theoretical_loss": 3.412543981181491,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012096888594666163,
+      "loss": 2.3033,
+      "theoretical_loss": 3.412543981181491,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012037781350482316,
+      "loss": 2.2906,
+      "theoretical_loss": 3.412112064147577,
+      "tokens_seen": 2133196800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011978674106298468,
+      "loss": 2.2847,
+      "theoretical_loss": 3.4116809955176617,
+      "tokens_seen": 2136473600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011919566862114621,
+      "loss": 2.3138,
+      "theoretical_loss": 3.4112507723278416,
+      "tokens_seen": 2139750400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011860459617930775,
+      "loss": 2.3131,
+      "theoretical_loss": 3.410821391629086,
+      "tokens_seen": 2143027200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011801352373746927,
+      "loss": 2.3053,
+      "theoretical_loss": 3.4103928504871406,
+      "tokens_seen": 2146304000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011742245129563079,
+      "loss": 2.3416,
+      "theoretical_loss": 3.4099651459824294,
+      "tokens_seen": 2149580800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011683137885379231,
+      "loss": 2.3522,
+      "theoretical_loss": 3.409538275209961,
+      "tokens_seen": 2152857600
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011624030641195385,
+      "loss": 2.3005,
+      "theoretical_loss": 3.409112235279233,
+      "tokens_seen": 2156134400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011564923397011538,
+      "loss": 2.2683,
+      "theoretical_loss": 3.4086870233141355,
+      "tokens_seen": 2159411200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001150581615282769,
+      "loss": 2.2678,
+      "theoretical_loss": 3.4082626364528625,
+      "tokens_seen": 2162688000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011446708908643844,
+      "loss": 2.2979,
+      "theoretical_loss": 3.4078390718478153,
+      "tokens_seen": 2165964800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011387601664459997,
+      "loss": 2.288,
+      "theoretical_loss": 3.4074163266655115,
+      "tokens_seen": 2169241600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011328494420276149,
+      "loss": 2.3404,
+      "theoretical_loss": 3.4069943980864945,
+      "tokens_seen": 2172518400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011269387176092302,
+      "loss": 2.3328,
+      "theoretical_loss": 3.4065732833052436,
+      "tokens_seen": 2175795200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011210279931908456,
+      "loss": 2.3568,
+      "theoretical_loss": 3.406152979530081,
+      "tokens_seen": 2179072000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011151172687724608,
+      "loss": 2.3747,
+      "theoretical_loss": 3.405733483983088,
+      "tokens_seen": 2182348800
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011092065443540761,
+      "loss": 2.397,
+      "theoretical_loss": 3.4053147939000117,
+      "tokens_seen": 2185625600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011032958199356912,
+      "loss": 2.3777,
+      "theoretical_loss": 3.4048969065301806,
+      "tokens_seen": 2188902400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010973850955173066,
+      "loss": 2.388,
+      "theoretical_loss": 3.4044798191364163,
+      "tokens_seen": 2192179200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010914743710989219,
+      "loss": 2.3678,
+      "theoretical_loss": 3.4040635289949472,
+      "tokens_seen": 2195456000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010855636466805371,
+      "loss": 2.3831,
+      "theoretical_loss": 3.403648033395324,
+      "tokens_seen": 2198732800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010796529222621525,
+      "loss": 2.367,
+      "theoretical_loss": 3.403233329640333,
+      "tokens_seen": 2202009600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010737421978437678,
+      "loss": 2.3233,
+      "theoretical_loss": 3.4028194150459137,
+      "tokens_seen": 2205286400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001067831473425383,
+      "loss": 2.3136,
+      "theoretical_loss": 3.4024062869410745,
+      "tokens_seen": 2208563200
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010619207490069983,
+      "loss": 2.3256,
+      "theoretical_loss": 3.4019939426678096,
+      "tokens_seen": 2211840000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010560100245886137,
+      "loss": 2.3463,
+      "theoretical_loss": 3.401582379581017,
+      "tokens_seen": 2215116800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010500993001702289,
+      "loss": 2.3453,
+      "theoretical_loss": 3.401171595048417,
+      "tokens_seen": 2218393600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010441885757518442,
+      "loss": 2.3347,
+      "theoretical_loss": 3.40076158645047,
+      "tokens_seen": 2221670400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010382778513334595,
+      "loss": 2.3302,
+      "theoretical_loss": 3.400352351180298,
+      "tokens_seen": 2224947200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010323671269150748,
+      "loss": 2.347,
+      "theoretical_loss": 3.3999438866436047,
+      "tokens_seen": 2228224000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.000102645640249669,
+      "loss": 2.3523,
+      "theoretical_loss": 3.399536190258594,
+      "tokens_seen": 2231500800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010205456780783052,
+      "loss": 2.3391,
+      "theoretical_loss": 3.399129259455895,
+      "tokens_seen": 2234777600
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010146349536599206,
+      "loss": 2.2914,
+      "theoretical_loss": 3.3987230916784803,
+      "tokens_seen": 2238054400
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010087242292415359,
+      "loss": 2.3093,
+      "theoretical_loss": 3.398317684381593,
+      "tokens_seen": 2241331200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010028135048231511,
+      "loss": 2.3195,
+      "theoretical_loss": 3.397913035032666,
+      "tokens_seen": 2244608000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.969027804047664e-05,
+      "loss": 2.3128,
+      "theoretical_loss": 3.3975091411112484,
+      "tokens_seen": 2247884800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.909920559863817e-05,
+      "loss": 2.3444,
+      "theoretical_loss": 3.397106000108929,
+      "tokens_seen": 2251161600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.85081331567997e-05,
+      "loss": 2.3649,
+      "theoretical_loss": 3.3967036095292613,
+      "tokens_seen": 2254438400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.791706071496123e-05,
+      "loss": 2.3858,
+      "theoretical_loss": 3.3963019668876897,
+      "tokens_seen": 2257715200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.732598827312276e-05,
+      "loss": 2.3611,
+      "theoretical_loss": 3.395901069711476,
+      "tokens_seen": 2260992000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.673491583128429e-05,
+      "loss": 2.372,
+      "theoretical_loss": 3.3955009155396247,
+      "tokens_seen": 2264268800
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.614384338944582e-05,
+      "loss": 2.3576,
+      "theoretical_loss": 3.3951015019228117,
+      "tokens_seen": 2267545600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.555277094760733e-05,
+      "loss": 2.3436,
+      "theoretical_loss": 3.3947028264233112,
+      "tokens_seen": 2270822400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.496169850576887e-05,
+      "loss": 2.356,
+      "theoretical_loss": 3.3943048866149255,
+      "tokens_seen": 2274099200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.438244751276718e-05,
+      "loss": 2.3392,
+      "theoretical_loss": 3.3939076800829118,
+      "tokens_seen": 2277376000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.379137507092869e-05,
+      "loss": 2.3636,
+      "theoretical_loss": 3.393511204423914,
+      "tokens_seen": 2280652800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.320030262909022e-05,
+      "loss": 2.3646,
+      "theoretical_loss": 3.3931154572458917,
+      "tokens_seen": 2283929600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.260923018725175e-05,
+      "loss": 2.3427,
+      "theoretical_loss": 3.3927204361680503,
+      "tokens_seen": 2287206400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.201815774541327e-05,
+      "loss": 2.3122,
+      "theoretical_loss": 3.3923261388207733,
+      "tokens_seen": 2290483200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.692480728193894,
+      "debugging/distinct-1-grams": 0.7869783046102373,
+      "debugging/distinct-2-grams": 0.9512166388994446,
+      "debugging/entropy-1-grams": 5.917774756532731,
+      "debugging/entropy-2-grams": 6.771509592332862,
+      "debugging/length": 703.7142857142857,
+      "debugging/num_segments": 14,
+      "epoch": 0.82,
+      "objective/train/docs_used": 1145810,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.2222933769226074,
+      "objective/train/theoretical_loss": 3.3919325628455534,
+      "objective/train/tokens_used": 2314220000,
+      "theoretical_loss": 3.3919325628455534,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.142708530357481e-05,
+      "loss": 2.2828,
+      "theoretical_loss": 3.3919325628455534,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.083601286173633e-05,
+      "loss": 2.3173,
+      "theoretical_loss": 3.3915397058949233,
+      "tokens_seen": 2297036800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.024494041989786e-05,
+      "loss": 2.3196,
+      "theoretical_loss": 3.3911475656323904,
+      "tokens_seen": 2300313600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.96538679780594e-05,
+      "loss": 2.3075,
+      "theoretical_loss": 3.3907561397323693,
+      "tokens_seen": 2303590400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.906279553622092e-05,
+      "loss": 2.2948,
+      "theoretical_loss": 3.3903654258801152,
+      "tokens_seen": 2306867200
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.847172309438245e-05,
+      "loss": 2.2845,
+      "theoretical_loss": 3.389975421771658,
+      "tokens_seen": 2310144000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.788065065254399e-05,
+      "loss": 2.2913,
+      "theoretical_loss": 3.389586125113736,
+      "tokens_seen": 2313420800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.728957821070551e-05,
+      "loss": 2.3035,
+      "theoretical_loss": 3.389197533623734,
+      "tokens_seen": 2316697600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.669850576886704e-05,
+      "loss": 2.313,
+      "theoretical_loss": 3.3888096450296152,
+      "tokens_seen": 2319974400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.610743332702855e-05,
+      "loss": 2.2911,
+      "theoretical_loss": 3.3884224570698613,
+      "tokens_seen": 2323251200
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.551636088519008e-05,
+      "loss": 2.2868,
+      "theoretical_loss": 3.388035967493405,
+      "tokens_seen": 2326528000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.492528844335162e-05,
+      "loss": 2.2606,
+      "theoretical_loss": 3.387650174059569,
+      "tokens_seen": 2329804800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.433421600151314e-05,
+      "loss": 2.2747,
+      "theoretical_loss": 3.3872650745380044,
+      "tokens_seen": 2333081600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.374314355967467e-05,
+      "loss": 2.2807,
+      "theoretical_loss": 3.3868806667086275,
+      "tokens_seen": 2336358400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.315207111783621e-05,
+      "loss": 2.2866,
+      "theoretical_loss": 3.3864969483615575,
+      "tokens_seen": 2339635200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.256099867599773e-05,
+      "loss": 2.2958,
+      "theoretical_loss": 3.3861139172970582,
+      "tokens_seen": 2342912000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.196992623415926e-05,
+      "loss": 2.3058,
+      "theoretical_loss": 3.385731571325474,
+      "tokens_seen": 2346188800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.13788537923208e-05,
+      "loss": 2.32,
+      "theoretical_loss": 3.385349908267171,
+      "tokens_seen": 2349465600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.078778135048232e-05,
+      "loss": 2.3319,
+      "theoretical_loss": 3.3849689259524784,
+      "tokens_seen": 2352742400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.019670890864385e-05,
+      "loss": 2.3438,
+      "theoretical_loss": 3.3845886222216275,
+      "tokens_seen": 2356019200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.960563646680537e-05,
+      "loss": 2.3222,
+      "theoretical_loss": 3.384208994924694,
+      "tokens_seen": 2359296000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.90145640249669e-05,
+      "loss": 2.3053,
+      "theoretical_loss": 3.3838300419215384,
+      "tokens_seen": 2362572800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.842349158312843e-05,
+      "loss": 2.3186,
+      "theoretical_loss": 3.383451761081749,
+      "tokens_seen": 2365849600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.783241914128995e-05,
+      "loss": 2.3211,
+      "theoretical_loss": 3.383074150284584,
+      "tokens_seen": 2369126400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.724134669945148e-05,
+      "loss": 2.2908,
+      "theoretical_loss": 3.3826972074189134,
+      "tokens_seen": 2372403200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.665027425761302e-05,
+      "loss": 2.3135,
+      "theoretical_loss": 3.3823209303831634,
+      "tokens_seen": 2375680000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.605920181577454e-05,
+      "loss": 2.3291,
+      "theoretical_loss": 3.3819453170852602,
+      "tokens_seen": 2378956800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.546812937393607e-05,
+      "loss": 2.3303,
+      "theoretical_loss": 3.381570365442573,
+      "tokens_seen": 2382233600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.48770569320976e-05,
+      "loss": 2.343,
+      "theoretical_loss": 3.3811960733818585,
+      "tokens_seen": 2385510400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.428598449025913e-05,
+      "loss": 2.3291,
+      "theoretical_loss": 3.380822438839207,
+      "tokens_seen": 2388787200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.369491204842066e-05,
+      "loss": 2.3437,
+      "theoretical_loss": 3.380449459759986,
+      "tokens_seen": 2392064000
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.310383960658218e-05,
+      "loss": 2.3417,
+      "theoretical_loss": 3.380077134098787,
+      "tokens_seen": 2395340800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.251276716474372e-05,
+      "loss": 2.3058,
+      "theoretical_loss": 3.3797054598193723,
+      "tokens_seen": 2398617600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.192169472290524e-05,
+      "loss": 2.3221,
+      "theoretical_loss": 3.379334434894619,
+      "tokens_seen": 2401894400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.133062228106676e-05,
+      "loss": 2.3,
+      "theoretical_loss": 3.378964057306467,
+      "tokens_seen": 2405171200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.07395498392283e-05,
+      "loss": 2.2979,
+      "theoretical_loss": 3.378594325045867,
+      "tokens_seen": 2408448000
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.014847739738983e-05,
+      "loss": 2.283,
+      "theoretical_loss": 3.378225236112729,
+      "tokens_seen": 2411724800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.955740495555135e-05,
+      "loss": 2.2943,
+      "theoretical_loss": 3.377856788515865,
+      "tokens_seen": 2415001600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.896633251371288e-05,
+      "loss": 2.3198,
+      "theoretical_loss": 3.3774889802729446,
+      "tokens_seen": 2418278400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.83752600718744e-05,
+      "loss": 2.3157,
+      "theoretical_loss": 3.377121809410438,
+      "tokens_seen": 2421555200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.778418763003594e-05,
+      "loss": 2.3359,
+      "theoretical_loss": 3.3767552739635684,
+      "tokens_seen": 2424832000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.719311518819747e-05,
+      "loss": 2.3097,
+      "theoretical_loss": 3.37638937197626,
+      "tokens_seen": 2428108800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.6602042746359e-05,
+      "loss": 2.3226,
+      "theoretical_loss": 3.3760241015010872,
+      "tokens_seen": 2431385600
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.601097030452053e-05,
+      "loss": 2.3401,
+      "theoretical_loss": 3.375659460599226,
+      "tokens_seen": 2434662400
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.541989786268206e-05,
+      "loss": 2.2883,
+      "theoretical_loss": 3.375295447340404,
+      "tokens_seen": 2437939200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.482882542084357e-05,
+      "loss": 2.2965,
+      "theoretical_loss": 3.3749320598028527,
+      "tokens_seen": 2441216000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.42377529790051e-05,
+      "loss": 2.2686,
+      "theoretical_loss": 3.3745692960732545,
+      "tokens_seen": 2444492800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.364668053716664e-05,
+      "loss": 2.3043,
+      "theoretical_loss": 3.374207154246699,
+      "tokens_seen": 2447769600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.305560809532816e-05,
+      "loss": 2.3253,
+      "theoretical_loss": 3.373845632426633,
+      "tokens_seen": 2451046400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.24645356534897e-05,
+      "loss": 2.3325,
+      "theoretical_loss": 3.373484728724812,
+      "tokens_seen": 2454323200
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/docs_used": 1229465,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.503016948699951,
+      "objective/train/theoretical_loss": 3.3731244412612527,
+      "objective/train/tokens_used": 2478060000,
+      "theoretical_loss": 3.3731244412612527,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.187346321165123e-05,
+      "loss": 2.34,
+      "theoretical_loss": 3.3731244412612527,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.128239076981275e-05,
+      "loss": 2.3452,
+      "theoretical_loss": 3.3727647681641884,
+      "tokens_seen": 2460876800
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0691318327974276e-05,
+      "loss": 2.3094,
+      "theoretical_loss": 3.3724057075700182,
+      "tokens_seen": 2464153600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.010024588613581e-05,
+      "loss": 2.3144,
+      "theoretical_loss": 3.372047257623265,
+      "tokens_seen": 2467430400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.950917344429733e-05,
+      "loss": 2.2948,
+      "theoretical_loss": 3.3716894164765256,
+      "tokens_seen": 2470707200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.891810100245886e-05,
+      "loss": 2.3295,
+      "theoretical_loss": 3.371332182290427,
+      "tokens_seen": 2473984000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.8327028560620387e-05,
+      "loss": 2.2853,
+      "theoretical_loss": 3.37097555323358,
+      "tokens_seen": 2477260800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.773595611878192e-05,
+      "loss": 2.32,
+      "theoretical_loss": 3.370619527482535,
+      "tokens_seen": 2480537600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.714488367694345e-05,
+      "loss": 2.3224,
+      "theoretical_loss": 3.370264103221736,
+      "tokens_seen": 2483814400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.6553811235104976e-05,
+      "loss": 2.3161,
+      "theoretical_loss": 3.369909278643477,
+      "tokens_seen": 2487091200
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.596273879326651e-05,
+      "loss": 2.3152,
+      "theoretical_loss": 3.369555051947857,
+      "tokens_seen": 2490368000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.537166635142803e-05,
+      "loss": 2.3328,
+      "theoretical_loss": 3.369201421342737,
+      "tokens_seen": 2493644800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.478059390958956e-05,
+      "loss": 2.3805,
+      "theoretical_loss": 3.3688483850436945,
+      "tokens_seen": 2496921600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4189521467751086e-05,
+      "loss": 2.4431,
+      "theoretical_loss": 3.368495941273984,
+      "tokens_seen": 2500198400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.359844902591262e-05,
+      "loss": 2.5094,
+      "theoretical_loss": 3.3681440882644886,
+      "tokens_seen": 2503475200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.300737658407415e-05,
+      "loss": 2.5461,
+      "theoretical_loss": 3.3677928242536828,
+      "tokens_seen": 2506752000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2416304142235676e-05,
+      "loss": 2.5254,
+      "theoretical_loss": 3.3674421474875853,
+      "tokens_seen": 2510028800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.1825231700397197e-05,
+      "loss": 2.488,
+      "theoretical_loss": 3.367092056219721,
+      "tokens_seen": 2513305600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.123415925855873e-05,
+      "loss": 2.4795,
+      "theoretical_loss": 3.3667425487110756,
+      "tokens_seen": 2516582400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.064308681672026e-05,
+      "loss": 2.4391,
+      "theoretical_loss": 3.366393623230057,
+      "tokens_seen": 2519859200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.0052014374881786e-05,
+      "loss": 2.4178,
+      "theoretical_loss": 3.366045278052451,
+      "tokens_seen": 2523136000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9460941933043314e-05,
+      "loss": 2.3715,
+      "theoretical_loss": 3.365697511461384,
+      "tokens_seen": 2526412800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.886986949120485e-05,
+      "loss": 2.3864,
+      "theoretical_loss": 3.365350321747279,
+      "tokens_seen": 2529689600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.827879704936637e-05,
+      "loss": 2.3774,
+      "theoretical_loss": 3.365003707207816,
+      "tokens_seen": 2532966400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7687724607527896e-05,
+      "loss": 2.3324,
+      "theoretical_loss": 3.3646576661478944,
+      "tokens_seen": 2536243200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.709665216568943e-05,
+      "loss": 2.3311,
+      "theoretical_loss": 3.3643121968795895,
+      "tokens_seen": 2539520000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.650557972385096e-05,
+      "loss": 2.3335,
+      "theoretical_loss": 3.3639672977221142,
+      "tokens_seen": 2542796800
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5914507282012486e-05,
+      "loss": 2.3127,
+      "theoretical_loss": 3.3636229670017808,
+      "tokens_seen": 2546073600
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.532343484017401e-05,
+      "loss": 2.2841,
+      "theoretical_loss": 3.3632792030519605,
+      "tokens_seen": 2549350400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.473236239833554e-05,
+      "loss": 2.3087,
+      "theoretical_loss": 3.362936004213045,
+      "tokens_seen": 2552627200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.414128995649707e-05,
+      "loss": 2.322,
+      "theoretical_loss": 3.362593368832409,
+      "tokens_seen": 2555904000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.3550217514658596e-05,
+      "loss": 2.3144,
+      "theoretical_loss": 3.36225129526437,
+      "tokens_seen": 2559180800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2959145072820124e-05,
+      "loss": 2.3066,
+      "theoretical_loss": 3.3619097818701515,
+      "tokens_seen": 2562457600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.236807263098166e-05,
+      "loss": 2.2959,
+      "theoretical_loss": 3.3615688270178454,
+      "tokens_seen": 2565734400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1777000189143186e-05,
+      "loss": 2.299,
+      "theoretical_loss": 3.3612284290823737,
+      "tokens_seen": 2569011200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1185927747304706e-05,
+      "loss": 2.2796,
+      "theoretical_loss": 3.3608885864454523,
+      "tokens_seen": 2572288000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0594855305466234e-05,
+      "loss": 2.2965,
+      "theoretical_loss": 3.360549297495553,
+      "tokens_seen": 2575564800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.000378286362777e-05,
+      "loss": 2.2991,
+      "theoretical_loss": 3.3602105606278663,
+      "tokens_seen": 2578841600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9412710421789296e-05,
+      "loss": 2.3,
+      "theoretical_loss": 3.359872374244266,
+      "tokens_seen": 2582118400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.883345942878759e-05,
+      "loss": 2.3169,
+      "theoretical_loss": 3.3595347367532744,
+      "tokens_seen": 2585395200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.824238698694912e-05,
+      "loss": 2.3561,
+      "theoretical_loss": 3.359197646570022,
+      "tokens_seen": 2588672000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.765131454511065e-05,
+      "loss": 2.3009,
+      "theoretical_loss": 3.3588611021162156,
+      "tokens_seen": 2591948800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.706024210327218e-05,
+      "loss": 2.3239,
+      "theoretical_loss": 3.3585251018201006,
+      "tokens_seen": 2595225600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6469169661433705e-05,
+      "loss": 2.3336,
+      "theoretical_loss": 3.358189644116427,
+      "tokens_seen": 2598502400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.587809721959524e-05,
+      "loss": 2.3532,
+      "theoretical_loss": 3.3578547274464134,
+      "tokens_seen": 2601779200
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.528702477775676e-05,
+      "loss": 2.4238,
+      "theoretical_loss": 3.357520350257713,
+      "tokens_seen": 2605056000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.469595233591829e-05,
+      "loss": 2.4898,
+      "theoretical_loss": 3.357186511004378,
+      "tokens_seen": 2608332800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.410487989407982e-05,
+      "loss": 2.4528,
+      "theoretical_loss": 3.356853208146826,
+      "tokens_seen": 2611609600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.351380745224135e-05,
+      "loss": 2.4105,
+      "theoretical_loss": 3.356520440151805,
+      "tokens_seen": 2614886400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.292273501040288e-05,
+      "loss": 2.4036,
+      "theoretical_loss": 3.3561882054923613,
+      "tokens_seen": 2618163200
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6940386524184803,
+      "debugging/distinct-1-grams": 0.8077683424419912,
+      "debugging/distinct-2-grams": 0.9727206942058755,
+      "debugging/entropy-1-grams": 6.015592698009554,
+      "debugging/entropy-2-grams": 6.835441617871684,
+      "debugging/length": 666.6875,
+      "debugging/num_segments": 16,
+      "epoch": 0.94,
+      "objective/train/docs_used": 1303318,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.270509719848633,
+      "objective/train/theoretical_loss": 3.3558565026478027,
+      "objective/train/tokens_used": 2641900000,
+      "theoretical_loss": 3.3558565026478027,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2331662568564405e-05,
+      "loss": 2.3789,
+      "theoretical_loss": 3.3558565026478027,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.174059012672593e-05,
+      "loss": 2.3605,
+      "theoretical_loss": 3.3555253301036685,
+      "tokens_seen": 2624716800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.114951768488746e-05,
+      "loss": 2.3602,
+      "theoretical_loss": 3.3551946863516937,
+      "tokens_seen": 2627993600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.055844524304899e-05,
+      "loss": 2.387,
+      "theoretical_loss": 3.3548645698897777,
+      "tokens_seen": 2631270400
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.996737280121052e-05,
+      "loss": 2.3478,
+      "theoretical_loss": 3.3545349792219494,
+      "tokens_seen": 2634547200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9376300359372043e-05,
+      "loss": 2.2972,
+      "theoretical_loss": 3.354205912858337,
+      "tokens_seen": 2637824000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8785227917533574e-05,
+      "loss": 2.3295,
+      "theoretical_loss": 3.353877369315133,
+      "tokens_seen": 2641100800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.81941554756951e-05,
+      "loss": 2.3328,
+      "theoretical_loss": 3.353549347114565,
+      "tokens_seen": 2644377600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.760308303385663e-05,
+      "loss": 2.3295,
+      "theoretical_loss": 3.3532218447848616,
+      "tokens_seen": 2647654400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7012010592018156e-05,
+      "loss": 2.2904,
+      "theoretical_loss": 3.352894860860221,
+      "tokens_seen": 2650931200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6420938150179687e-05,
+      "loss": 2.3452,
+      "theoretical_loss": 3.352568393880779,
+      "tokens_seen": 2654208000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5829865708341215e-05,
+      "loss": 2.3384,
+      "theoretical_loss": 3.3522424423925794,
+      "tokens_seen": 2657484800
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5238793266502742e-05,
+      "loss": 2.3238,
+      "theoretical_loss": 3.3519170049475413,
+      "tokens_seen": 2660761600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4647720824664273e-05,
+      "loss": 2.301,
+      "theoretical_loss": 3.351592080103429,
+      "tokens_seen": 2664038400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4056648382825798e-05,
+      "loss": 2.333,
+      "theoretical_loss": 3.3512676664238192,
+      "tokens_seen": 2667315200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.346557594098733e-05,
+      "loss": 2.373,
+      "theoretical_loss": 3.350943762478075,
+      "tokens_seen": 2670592000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2874503499148856e-05,
+      "loss": 2.3555,
+      "theoretical_loss": 3.3506203668413095,
+      "tokens_seen": 2673868800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2283431057310387e-05,
+      "loss": 2.3812,
+      "theoretical_loss": 3.3502974780943613,
+      "tokens_seen": 2677145600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.169235861547191e-05,
+      "loss": 2.3417,
+      "theoretical_loss": 3.34997509482376,
+      "tokens_seen": 2680422400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1101286173633442e-05,
+      "loss": 2.3311,
+      "theoretical_loss": 3.3496532156217005,
+      "tokens_seen": 2683699200
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.051021373179497e-05,
+      "loss": 2.308,
+      "theoretical_loss": 3.34933183908601,
+      "tokens_seen": 2686976000
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9919141289956497e-05,
+      "loss": 2.3075,
+      "theoretical_loss": 3.3490109638201195,
+      "tokens_seen": 2690252800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9328068848118025e-05,
+      "loss": 2.3478,
+      "theoretical_loss": 3.348690588433038,
+      "tokens_seen": 2693529600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8736996406279556e-05,
+      "loss": 2.3191,
+      "theoretical_loss": 3.3483707115393164,
+      "tokens_seen": 2696806400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.814592396444108e-05,
+      "loss": 2.3375,
+      "theoretical_loss": 3.3480513317590264,
+      "tokens_seen": 2700083200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.755485152260261e-05,
+      "loss": 2.3652,
+      "theoretical_loss": 3.347732447717727,
+      "tokens_seen": 2703360000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6963779080764142e-05,
+      "loss": 2.3631,
+      "theoretical_loss": 3.347414058046437,
+      "tokens_seen": 2706636800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6372706638925666e-05,
+      "loss": 2.3482,
+      "theoretical_loss": 3.3470961613816086,
+      "tokens_seen": 2709913600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5781634197087197e-05,
+      "loss": 2.3422,
+      "theoretical_loss": 3.3467787563650955,
+      "tokens_seen": 2713190400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5190561755248723e-05,
+      "loss": 2.3193,
+      "theoretical_loss": 3.34646184164413,
+      "tokens_seen": 2716467200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4599489313410252e-05,
+      "loss": 2.3673,
+      "theoretical_loss": 3.346145415871291,
+      "tokens_seen": 2719744000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.400841687157178e-05,
+      "loss": 2.3431,
+      "theoretical_loss": 3.345829477704478,
+      "tokens_seen": 2723020800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3417344429733307e-05,
+      "loss": 2.3226,
+      "theoretical_loss": 3.345514025806884,
+      "tokens_seen": 2726297600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2826271987894837e-05,
+      "loss": 2.2864,
+      "theoretical_loss": 3.3451990588469687,
+      "tokens_seen": 2729574400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2235199546056366e-05,
+      "loss": 2.2984,
+      "theoretical_loss": 3.3448845754984307,
+      "tokens_seen": 2732851200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1644127104217893e-05,
+      "loss": 2.2937,
+      "theoretical_loss": 3.344570574440179,
+      "tokens_seen": 2736128000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1053054662379423e-05,
+      "loss": 2.3087,
+      "theoretical_loss": 3.3442570543563104,
+      "tokens_seen": 2739404800
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.046198222054095e-05,
+      "loss": 2.3125,
+      "theoretical_loss": 3.34394401393608,
+      "tokens_seen": 2742681600
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.870909778702478e-06,
+      "loss": 2.3205,
+      "theoretical_loss": 3.343631451873874,
+      "tokens_seen": 2745958400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.279837336864007e-06,
+      "loss": 2.3203,
+      "theoretical_loss": 3.343319366869187,
+      "tokens_seen": 2749235200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.688764895025535e-06,
+      "loss": 2.3166,
+      "theoretical_loss": 3.343007757626593,
+      "tokens_seen": 2752512000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.097692453187062e-06,
+      "loss": 2.319,
+      "theoretical_loss": 3.34269662285572,
+      "tokens_seen": 2755788800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.506620011348591e-06,
+      "loss": 2.311,
+      "theoretical_loss": 3.342385961271226,
+      "tokens_seen": 2759065600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.915547569510119e-06,
+      "loss": 2.3098,
+      "theoretical_loss": 3.3420757715927705,
+      "tokens_seen": 2762342400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.324475127671647e-06,
+      "loss": 2.2793,
+      "theoretical_loss": 3.341766052544993,
+      "tokens_seen": 2765619200
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.733402685833176e-06,
+      "loss": 2.299,
+      "theoretical_loss": 3.341456802857484,
+      "tokens_seen": 2768896000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.142330243994704e-06,
+      "loss": 2.3221,
+      "theoretical_loss": 3.341148021264762,
+      "tokens_seen": 2772172800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.551257802156233e-06,
+      "loss": 2.3195,
+      "theoretical_loss": 3.3408397065062485,
+      "tokens_seen": 2775449600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.960185360317761e-06,
+      "loss": 2.3145,
+      "theoretical_loss": 3.3405318573262424,
+      "tokens_seen": 2778726400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.369112918479289e-06,
+      "loss": 2.3171,
+      "theoretical_loss": 3.3402244724738965,
+      "tokens_seen": 2782003200
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/docs_used": 1381756,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.206209182739258,
+      "objective/train/theoretical_loss": 3.3399175507031913,
+      "objective/train/tokens_used": 2805740000,
+      "theoretical_loss": 3.3399175507031913,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.778040476640817e-06,
+      "loss": 2.2985,
+      "theoretical_loss": 3.3399175507031913,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.1869680348023455e-06,
+      "loss": 2.3251,
+      "theoretical_loss": 3.3396110907729133,
+      "tokens_seen": 2788556800
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.5958955929638737e-06,
+      "loss": 2.3196,
+      "theoretical_loss": 3.3393050914466285,
+      "tokens_seen": 2791833600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0048231511254019e-06,
+      "loss": 2.3389,
+      "theoretical_loss": 3.33899955149266,
+      "tokens_seen": 2795110400
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.137507092869302e-07,
+      "loss": 2.2961,
+      "theoretical_loss": 3.338694469684062,
+      "tokens_seen": 2798387200
+    }
+  ],
+  "max_steps": 42724,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.428920419221504e+18,
+  "trial_name": null,
+  "trial_params": null
+}