diff --git "a/checkpoint-42724/trainer_state.json" "b/checkpoint-42724/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-42724/trainer_state.json"
@@ -0,0 +1,15493 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "global_step": 42724,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1682242990654204e-06,
+      "loss": 10.8658,
+      "theoretical_loss": 20.812802717066116,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.336448598130841e-05,
+      "loss": 9.7159,
+      "theoretical_loss": 10.363548859282295,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.672897196261682e-05,
+      "loss": 8.8339,
+      "theoretical_loss": 8.96027440542832,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.009345794392523e-05,
+      "loss": 7.9042,
+      "theoretical_loss": 8.257160837770828,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.345794392523364e-05,
+      "loss": 7.0675,
+      "theoretical_loss": 7.80455017047313,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011682242990654206,
+      "loss": 6.5675,
+      "theoretical_loss": 7.477743633566995,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014018691588785047,
+      "loss": 6.3483,
+      "theoretical_loss": 7.225472152031461,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016355140186915886,
+      "loss": 6.0955,
+      "theoretical_loss": 7.022000360175642,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018691588785046728,
+      "loss": 5.9627,
+      "theoretical_loss": 6.852706072125594,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002102803738317757,
+      "loss": 5.7575,
+      "theoretical_loss": 6.7085424020813775,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023364485981308412,
+      "loss": 5.6734,
+      "theoretical_loss": 6.583551143945378,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002570093457943925,
+      "loss": 5.5759,
+      "theoretical_loss": 6.4736158466608344,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028037383177570094,
+      "loss": 5.4962,
+      "theoretical_loss": 6.375782617839931,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030373831775700936,
+      "loss": 5.387,
+      "theoretical_loss": 6.287865514264695,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003271028037383177,
+      "loss": 5.31,
+      "theoretical_loss": 6.208205077800142,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035046728971962614,
+      "loss": 5.2546,
+      "theoretical_loss": 6.13551414745929,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037383177570093456,
+      "loss": 5.196,
+      "theoretical_loss": 6.068775822887412,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000397196261682243,
+      "loss": 5.1134,
+      "theoretical_loss": 6.007173857995872,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004205607476635514,
+      "loss": 5.0982,
+      "theoretical_loss": 5.950043919222669,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004439252336448598,
+      "loss": 5.0706,
+      "theoretical_loss": 5.89683866811525,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046728971962616824,
+      "loss": 4.9896,
+      "theoretical_loss": 5.847102241784889,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004906542056074767,
+      "loss": 4.8916,
+      "theoretical_loss": 5.80045126829951,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998581426139588,
+      "loss": 4.9238,
+      "theoretical_loss": 5.756560518460697,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996217136372233,
+      "loss": 4.8673,
+      "theoretical_loss": 5.715151906614886,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499385284660488,
+      "loss": 4.7589,
+      "theoretical_loss": 5.675985950013518,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991488556837526,
+      "loss": 4.7546,
+      "theoretical_loss": 5.638855059590318,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989124267070172,
+      "loss": 4.6908,
+      "theoretical_loss": 5.603578213244772,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986759977302818,
+      "loss": 4.7011,
+      "theoretical_loss": 5.569996685496657,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984395687535465,
+      "loss": 4.6645,
+      "theoretical_loss": 5.53797059334095,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498203139776811,
+      "loss": 4.5751,
+      "theoretical_loss": 5.507376079224074,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979667108000757,
+      "loss": 4.493,
+      "theoretical_loss": 5.478102996075402,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977302818233403,
+      "loss": 4.5319,
+      "theoretical_loss": 5.450052991438716,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974938528466049,
+      "loss": 4.4256,
+      "theoretical_loss": 5.4231379114505165,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972574238698695,
+      "loss": 4.4082,
+      "theoretical_loss": 5.397278463098006,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970209948931342,
+      "loss": 4.4953,
+      "theoretical_loss": 5.372403086519811,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967845659163987,
+      "loss": 4.3646,
+      "theoretical_loss": 5.348446999254799,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965481369396634,
+      "loss": 4.4128,
+      "theoretical_loss": 5.3253513821291545,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496311707962928,
+      "loss": 4.3023,
+      "theoretical_loss": 5.303062682496617,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960752789861926,
+      "loss": 4.3546,
+      "theoretical_loss": 5.281532015245618,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958388500094572,
+      "loss": 4.2839,
+      "theoretical_loss": 5.260714645678606,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956024210327218,
+      "loss": 4.2506,
+      "theoretical_loss": 5.240569541288942,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953659920559864,
+      "loss": 4.2113,
+      "theoretical_loss": 5.221058981785759,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495129563079251,
+      "loss": 4.2045,
+      "theoretical_loss": 5.202148218579597,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948931341025156,
+      "loss": 4.1605,
+      "theoretical_loss": 5.1838051764423945,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946567051257802,
+      "loss": 4.1034,
+      "theoretical_loss": 5.166000191271213,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944202761490448,
+      "loss": 4.1023,
+      "theoretical_loss": 5.148705778875405,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941838471723094,
+      "loss": 4.015,
+      "theoretical_loss": 5.131896430517482,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493947418195574,
+      "loss": 3.9781,
+      "theoretical_loss": 5.1155484316045445,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937109892188386,
+      "loss": 3.9207,
+      "theoretical_loss": 5.099639700477905,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934745602421033,
+      "loss": 3.8888,
+      "theoretical_loss": 5.084149644705477,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 46774,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.873431921005249,
+      "objective/train/theoretical_loss": 5.06905903266221,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.06905903266221,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932381312653678,
+      "loss": 3.7964,
+      "theoretical_loss": 5.06905903266221,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930017022886325,
+      "loss": 3.761,
+      "theoretical_loss": 5.054349878502332,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927652733118971,
+      "loss": 3.8266,
+      "theoretical_loss": 5.04000533889449,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925288443351617,
+      "loss": 3.7863,
+      "theoretical_loss": 5.02600962011627,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922924153584263,
+      "loss": 3.7525,
+      "theoretical_loss": 5.012347894295022,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492055986381691,
+      "loss": 3.7151,
+      "theoretical_loss": 4.999006223743629,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918195574049555,
+      "loss": 3.7239,
+      "theoretical_loss": 4.985971492477399,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915831284282202,
+      "loss": 3.7674,
+      "theoretical_loss": 4.973231344115766,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913466994514848,
+      "loss": 3.6962,
+      "theoretical_loss": 4.960774125473073,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911102704747494,
+      "loss": 3.5964,
+      "theoretical_loss": 4.948588835229052,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490873841498014,
+      "loss": 3.7099,
+      "theoretical_loss": 4.9366650771441005,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906374125212787,
+      "loss": 3.5834,
+      "theoretical_loss": 4.924993017348572,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904009835445432,
+      "loss": 3.6168,
+      "theoretical_loss": 4.913563345291011,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901645545678079,
+      "loss": 3.6325,
+      "theoretical_loss": 4.90236723797842,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899281255910725,
+      "loss": 3.6213,
+      "theoretical_loss": 4.891396327183692,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896916966143371,
+      "loss": 3.5461,
+      "theoretical_loss": 4.880642669331891,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894552676376017,
+      "loss": 3.6126,
+      "theoretical_loss": 4.8700987178090696,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892188386608663,
+      "loss": 3.5591,
+      "theoretical_loss": 4.859757297465288,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889824096841309,
+      "loss": 3.5483,
+      "theoretical_loss": 4.84961158110812,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887459807073955,
+      "loss": 3.5316,
+      "theoretical_loss": 4.839655067804491,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885095517306601,
+      "loss": 3.4951,
+      "theoretical_loss": 4.829881562827778,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882731227539247,
+      "loss": 3.4582,
+      "theoretical_loss": 4.820285159103877,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048803669377718936,
+      "loss": 3.4803,
+      "theoretical_loss": 4.810860220024846,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048780026480045396,
+      "loss": 3.4445,
+      "theoretical_loss": 4.801601363511853,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048756383582371855,
+      "loss": 3.4563,
+      "theoretical_loss": 4.792503447220913,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873274068469832,
+      "loss": 3.5527,
+      "theoretical_loss": 4.783561554795222,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870909778702478,
+      "loss": 3.4383,
+      "theoretical_loss": 4.774770983077213,
+      "tokens_seen": 99614720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868545488935124,
+      "loss": 3.5052,
+      "theoretical_loss": 4.766127230201681,
+      "tokens_seen": 100925440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486618119916777,
+      "loss": 3.4738,
+      "theoretical_loss": 4.757625984498683,
+      "tokens_seen": 102236160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048638169094004165,
+      "loss": 3.5177,
+      "theoretical_loss": 4.7492631141415815,
+      "tokens_seen": 103546880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861452619633062,
+      "loss": 3.4476,
+      "theoretical_loss": 4.741034657481437,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048590883298657084,
+      "loss": 3.4467,
+      "theoretical_loss": 4.732936814014305,
+      "tokens_seen": 106168320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856724040098355,
+      "loss": 3.4405,
+      "theoretical_loss": 4.724965935932751,
+      "tokens_seen": 107479040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048543597503310004,
+      "loss": 3.4147,
+      "theoretical_loss": 4.717118520217186,
+      "tokens_seen": 108789760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851995460563647,
+      "loss": 3.4864,
+      "theoretical_loss": 4.709391201226483,
+      "tokens_seen": 110100480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849631170796293,
+      "loss": 3.4632,
+      "theoretical_loss": 4.70178074375083,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847266881028939,
+      "loss": 3.5115,
+      "theoretical_loss": 4.694284036492922,
+      "tokens_seen": 112721920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048449025912615854,
+      "loss": 3.4733,
+      "theoretical_loss": 4.686898085946441,
+      "tokens_seen": 114032640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048425383014942313,
+      "loss": 3.4572,
+      "theoretical_loss": 4.6796200106433545,
+      "tokens_seen": 115343360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048401740117268773,
+      "loss": 3.4342,
+      "theoretical_loss": 4.6724470357438985,
+      "tokens_seen": 116654080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048378097219595233,
+      "loss": 3.4469,
+      "theoretical_loss": 4.665376487945246,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483544543219217,
+      "loss": 3.3911,
+      "theoretical_loss": 4.658405790686763,
+      "tokens_seen": 119275520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833081142424816,
+      "loss": 3.3665,
+      "theoretical_loss": 4.6515324596315395,
+      "tokens_seen": 120586240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830716852657462,
+      "loss": 3.3685,
+      "theoretical_loss": 4.6447540984054605,
+      "tokens_seen": 121896960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048283525628901083,
+      "loss": 3.4102,
+      "theoretical_loss": 4.638068394576544,
+      "tokens_seen": 123207680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048259882731227537,
+      "loss": 3.4189,
+      "theoretical_loss": 4.631473115858608,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048236239833554,
+      "loss": 3.4026,
+      "theoretical_loss": 4.624966106524518,
+      "tokens_seen": 125829120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821259693588047,
+      "loss": 3.3896,
+      "theoretical_loss": 4.61854528401543,
+      "tokens_seen": 127139840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818895403820692,
+      "loss": 3.4057,
+      "theoretical_loss": 4.6122086357333725,
+      "tokens_seen": 128450560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048165311140533387,
+      "loss": 3.3771,
+      "theoretical_loss": 4.605954216005544,
+      "tokens_seen": 129761280
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 82490,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.441986322402954,
+      "objective/train/theoretical_loss": 4.599780143209469,
+      "objective/train/tokens_used": 151532000,
+      "theoretical_loss": 4.599780143209469,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141668242859847,
+      "loss": 3.3107,
+      "theoretical_loss": 4.599780143209469,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048118025345186306,
+      "loss": 3.3261,
+      "theoretical_loss": 4.593684597048995,
+      "tokens_seen": 132382720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809438244751277,
+      "loss": 3.3271,
+      "theoretical_loss": 4.587665815971789,
+      "tokens_seen": 133693440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807073954983923,
+      "loss": 3.3729,
+      "theoretical_loss": 4.581722094719703,
+      "tokens_seen": 135004160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804709665216569,
+      "loss": 3.3567,
+      "theoretical_loss": 4.575851782003932,
+      "tokens_seen": 136314880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802345375449215,
+      "loss": 3.3513,
+      "theoretical_loss": 4.570053278297481,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047999810856818616,
+      "loss": 3.2978,
+      "theoretical_loss": 4.564325033737986,
+      "tokens_seen": 138936320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797616795914507,
+      "loss": 3.3,
+      "theoretical_loss": 4.558665546134362,
+      "tokens_seen": 140247040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047952525061471536,
+      "loss": 3.362,
+      "theoretical_loss": 4.5530733590712344,
+      "tokens_seen": 141557760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047928882163798,
+      "loss": 3.313,
+      "theoretical_loss": 4.547547060105504,
+      "tokens_seen": 142868480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905239266124455,
+      "loss": 3.3083,
+      "theoretical_loss": 4.542085279049763,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788159636845092,
+      "loss": 3.2759,
+      "theoretical_loss": 4.53668668633763,
+      "tokens_seen": 145489920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047857953470777375,
+      "loss": 3.264,
+      "theoretical_loss": 4.531349991466399,
+      "tokens_seen": 146800640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783431057310384,
+      "loss": 3.286,
+      "theoretical_loss": 4.526073941512697,
+      "tokens_seen": 148111360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047810667675430305,
+      "loss": 3.3484,
+      "theoretical_loss": 4.520857319717103,
+      "tokens_seen": 149422080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778702477775676,
+      "loss": 3.2817,
+      "theoretical_loss": 4.515698944133961,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047763381880083224,
+      "loss": 3.2996,
+      "theoretical_loss": 4.510597666342842,
+      "tokens_seen": 152043520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047739738982409684,
+      "loss": 3.2335,
+      "theoretical_loss": 4.5055523702183375,
+      "tokens_seen": 153354240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047716096084736144,
+      "loss": 3.2571,
+      "theoretical_loss": 4.500561970755069,
+      "tokens_seen": 154664960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769245318706261,
+      "loss": 3.2895,
+      "theoretical_loss": 4.495625412944987,
+      "tokens_seen": 155975680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766881028938907,
+      "loss": 3.3261,
+      "theoretical_loss": 4.490741670704231,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764516739171553,
+      "loss": 3.3318,
+      "theoretical_loss": 4.485909745846939,
+      "tokens_seen": 158597120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762152449404199,
+      "loss": 3.2679,
+      "theoretical_loss": 4.481128667103624,
+      "tokens_seen": 159907840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047597881596368453,
+      "loss": 3.2825,
+      "theoretical_loss": 4.476397489181794,
+      "tokens_seen": 161218560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047574238698694913,
+      "loss": 3.2408,
+      "theoretical_loss": 4.471715291866703,
+      "tokens_seen": 162529280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047550595801021373,
+      "loss": 3.2514,
+      "theoretical_loss": 4.467081179160184,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752695290334784,
+      "loss": 3.3081,
+      "theoretical_loss": 4.462494278455688,
+      "tokens_seen": 165150720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750331000567429,
+      "loss": 3.267,
+      "theoretical_loss": 4.457953739747708,
+      "tokens_seen": 166461440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747966710800076,
+      "loss": 3.1616,
+      "theoretical_loss": 4.453458734873907,
+      "tokens_seen": 167772160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745602421032722,
+      "loss": 3.2412,
+      "theoretical_loss": 4.449008456788359,
+      "tokens_seen": 169082880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432381312653677,
+      "loss": 3.3008,
+      "theoretical_loss": 4.4446021188643945,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740873841498014,
+      "loss": 3.2234,
+      "theoretical_loss": 4.440238954225603,
+      "tokens_seen": 171704320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473850955173066,
+      "loss": 3.2253,
+      "theoretical_loss": 4.435918215103694,
+      "tokens_seen": 173015040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736145261963306,
+      "loss": 3.2172,
+      "theoretical_loss": 4.431639172221903,
+      "tokens_seen": 174325760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733780972195952,
+      "loss": 3.186,
+      "theoretical_loss": 4.427401114202768,
+      "tokens_seen": 175636480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047314166824285987,
+      "loss": 3.1797,
+      "theoretical_loss": 4.4232033469991245,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047290523926612446,
+      "loss": 3.199,
+      "theoretical_loss": 4.419045193347259,
+      "tokens_seen": 178257920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047266881028938906,
+      "loss": 3.1988,
+      "theoretical_loss": 4.414925992241184,
+      "tokens_seen": 179568640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724323813126537,
+      "loss": 3.1514,
+      "theoretical_loss": 4.410845098427085,
+      "tokens_seen": 180879360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047219595233591826,
+      "loss": 3.1928,
+      "theoretical_loss": 4.406801881917029,
+      "tokens_seen": 182190080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719595233591829,
+      "loss": 3.1594,
+      "theoretical_loss": 4.402795727521044,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047172309438244756,
+      "loss": 3.1795,
+      "theoretical_loss": 4.398826034396784,
+      "tokens_seen": 184811520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714866654057121,
+      "loss": 3.1768,
+      "theoretical_loss": 4.394892215615966,
+      "tokens_seen": 186122240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047125023642897675,
+      "loss": 3.1642,
+      "theoretical_loss": 4.390993697746873,
+      "tokens_seen": 187432960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047101380745224135,
+      "loss": 3.2163,
+      "theoretical_loss": 4.387129920452183,
+      "tokens_seen": 188743680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047077737847550595,
+      "loss": 3.162,
+      "theoretical_loss": 4.383300336101497,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705409494987706,
+      "loss": 3.1984,
+      "theoretical_loss": 4.379504409397913,
+      "tokens_seen": 191365120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703045205220352,
+      "loss": 3.1694,
+      "theoretical_loss": 4.375741617018028,
+      "tokens_seen": 192675840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700680915452998,
+      "loss": 3.2412,
+      "theoretical_loss": 4.372011447264842,
+      "tokens_seen": 193986560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698316625685644,
+      "loss": 3.2147,
+      "theoretical_loss": 4.368313399732967,
+      "tokens_seen": 195297280
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 119577,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7380270957946777,
+      "objective/train/theoretical_loss": 4.364646984985669,
+      "objective/train/tokens_used": 217068000,
+      "theoretical_loss": 4.364646984985669,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046959523359182905,
+      "loss": 3.1619,
+      "theoretical_loss": 4.364646984985669,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693588046150936,
+      "loss": 3.1615,
+      "theoretical_loss": 4.3610117242432125,
+      "tokens_seen": 197918720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046912237563835824,
+      "loss": 3.1752,
+      "theoretical_loss": 4.357407149082071,
+      "tokens_seen": 199229440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688859466616229,
+      "loss": 3.1492,
+      "theoretical_loss": 4.353832801144535,
+      "tokens_seen": 200540160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046864951768488744,
+      "loss": 3.1317,
+      "theoretical_loss": 4.350288231858301,
+      "tokens_seen": 201850880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684130887081521,
+      "loss": 3.1784,
+      "theoretical_loss": 4.346773002165636,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681766597314167,
+      "loss": 3.1369,
+      "theoretical_loss": 4.343286682261728,
+      "tokens_seen": 204472320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679402307546813,
+      "loss": 3.1272,
+      "theoretical_loss": 4.339828851341859,
+      "tokens_seen": 205783040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046770380177794593,
+      "loss": 3.196,
+      "theoretical_loss": 4.336399097357047,
+      "tokens_seen": 207093760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046746737280121053,
+      "loss": 3.1771,
+      "theoretical_loss": 4.332997016777814,
+      "tokens_seen": 208404480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046723094382447513,
+      "loss": 3.1667,
+      "theoretical_loss": 4.329622214365776,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669945148477397,
+      "loss": 3.1952,
+      "theoretical_loss": 4.326274302952741,
+      "tokens_seen": 211025920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667580858710044,
+      "loss": 3.1607,
+      "theoretical_loss": 4.322952903227008,
+      "tokens_seen": 212336640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466521656894269,
+      "loss": 3.1631,
+      "theoretical_loss": 4.319657643526625,
+      "tokens_seen": 213647360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046628522791753357,
+      "loss": 3.1079,
+      "theoretical_loss": 4.316388159639304,
+      "tokens_seen": 214958080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660487989407982,
+      "loss": 3.1118,
+      "theoretical_loss": 4.313144094608757,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046581236996406277,
+      "loss": 3.1383,
+      "theoretical_loss": 4.3099250985472,
+      "tokens_seen": 217579520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655759409873274,
+      "loss": 3.1002,
+      "theoretical_loss": 4.306730828453813,
+      "tokens_seen": 218890240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046533951201059207,
+      "loss": 3.125,
+      "theoretical_loss": 4.303560948038896,
+      "tokens_seen": 220200960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651030830338566,
+      "loss": 3.0721,
+      "theoretical_loss": 4.300415127553551,
+      "tokens_seen": 221511680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046486665405712127,
+      "loss": 3.13,
+      "theoretical_loss": 4.297293043624652,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046463022508038586,
+      "loss": 3.1433,
+      "theoretical_loss": 4.294194379094925,
+      "tokens_seen": 224133120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046439379610365046,
+      "loss": 3.1008,
+      "theoretical_loss": 4.291118822867954,
+      "tokens_seen": 225443840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641573671269151,
+      "loss": 3.206,
+      "theoretical_loss": 4.288066069757916,
+      "tokens_seen": 226754560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639209381501797,
+      "loss": 3.1231,
+      "theoretical_loss": 4.285035820343889,
+      "tokens_seen": 228065280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636845091734443,
+      "loss": 3.1608,
+      "theoretical_loss": 4.28202778082858,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634480801967089,
+      "loss": 3.1397,
+      "theoretical_loss": 4.27904166290127,
+      "tokens_seen": 230686720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046321165121997356,
+      "loss": 3.1775,
+      "theoretical_loss": 4.276077183604883,
+      "tokens_seen": 231997440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629752222432381,
+      "loss": 3.1744,
+      "theoretical_loss": 4.2731340652069925,
+      "tokens_seen": 233308160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046273879326650275,
+      "loss": 3.1164,
+      "theoretical_loss": 4.270212035074646,
+      "tokens_seen": 234618880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625023642897674,
+      "loss": 3.2551,
+      "theoretical_loss": 4.267310825552862,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046226593531303195,
+      "loss": 3.1724,
+      "theoretical_loss": 4.264430173846689,
+      "tokens_seen": 237240320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620295063362966,
+      "loss": 3.1622,
+      "theoretical_loss": 4.261569821906684,
+      "tokens_seen": 238551040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617930773595612,
+      "loss": 3.1072,
+      "theoretical_loss": 4.258729516317703,
+      "tokens_seen": 239861760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615566483828258,
+      "loss": 3.1874,
+      "theoretical_loss": 4.255909008190896,
+      "tokens_seen": 241172480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046132021940609044,
+      "loss": 3.0979,
+      "theoretical_loss": 4.253108053058778,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046108379042935504,
+      "loss": 3.0572,
+      "theoretical_loss": 4.250326410773302,
+      "tokens_seen": 243793920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046084736145261964,
+      "loss": 3.0634,
+      "theoretical_loss": 4.247563845406793,
+      "tokens_seen": 245104640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046061093247588424,
+      "loss": 3.0988,
+      "theoretical_loss": 4.2448201251556945,
+      "tokens_seen": 246415360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603745034991489,
+      "loss": 3.0756,
+      "theoretical_loss": 4.242095022246987,
+      "tokens_seen": 247726080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601380745224135,
+      "loss": 3.1142,
+      "theoretical_loss": 4.239388312847225,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599016455456781,
+      "loss": 3.081,
+      "theoretical_loss": 4.236699776974095,
+      "tokens_seen": 250347520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045966521656894274,
+      "loss": 3.1122,
+      "theoretical_loss": 4.234029198410394,
+      "tokens_seen": 251658240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594287875922073,
+      "loss": 3.1194,
+      "theoretical_loss": 4.23137636462039,
+      "tokens_seen": 252968960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045919235861547193,
+      "loss": 3.0914,
+      "theoretical_loss": 4.228741066668439,
+      "tokens_seen": 254279680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589559296387366,
+      "loss": 3.1158,
+      "theoretical_loss": 4.226123099139831,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587195006620011,
+      "loss": 3.1033,
+      "theoretical_loss": 4.223522260063753,
+      "tokens_seen": 256901120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584830716852658,
+      "loss": 3.0847,
+      "theoretical_loss": 4.220938350838335,
+      "tokens_seen": 258211840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582466427085304,
+      "loss": 3.1236,
+      "theoretical_loss": 4.218371176157692,
+      "tokens_seen": 259522560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045801021373179497,
+      "loss": 3.0795,
+      "theoretical_loss": 4.215820543940906,
+      "tokens_seen": 260833280
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 155958,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.078427791595459,
+      "objective/train/theoretical_loss": 4.213286265262889,
+      "objective/train/tokens_used": 282604000,
+      "theoretical_loss": 4.213286265262889,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577737847550596,
+      "loss": 3.0425,
+      "theoretical_loss": 4.213286265262889,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575373557783242,
+      "loss": 3.0468,
+      "theoretical_loss": 4.210768154287058,
+      "tokens_seen": 263454720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573009268015888,
+      "loss": 3.0588,
+      "theoretical_loss": 4.208266028199782,
+      "tokens_seen": 264765440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570644978248534,
+      "loss": 3.0243,
+      "theoretical_loss": 4.205779707146535,
+      "tokens_seen": 266076160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045682806884811807,
+      "loss": 2.9936,
+      "theoretical_loss": 4.203309014169696,
+      "tokens_seen": 267386880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565916398713826,
+      "loss": 3.0404,
+      "theoretical_loss": 4.200853775147967,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045635521089464726,
+      "loss": 3.0029,
+      "theoretical_loss": 4.198413818737336,
+      "tokens_seen": 270008320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561187819179119,
+      "loss": 3.025,
+      "theoretical_loss": 4.195988976313554,
+      "tokens_seen": 271319040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045588235294117646,
+      "loss": 2.9815,
+      "theoretical_loss": 4.193579081916074,
+      "tokens_seen": 272629760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556459239644411,
+      "loss": 3.0173,
+      "theoretical_loss": 4.191183972193405,
+      "tokens_seen": 273940480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554094949877057,
+      "loss": 3.0058,
+      "theoretical_loss": 4.1888034863498484,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551730660109703,
+      "loss": 2.9504,
+      "theoretical_loss": 4.186437466093564,
+      "tokens_seen": 276561920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045493663703423496,
+      "loss": 2.9439,
+      "theoretical_loss": 4.1840857555859365,
+      "tokens_seen": 277872640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547002080574995,
+      "loss": 2.9548,
+      "theoretical_loss": 4.181748201392195,
+      "tokens_seen": 279183360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045446377908076415,
+      "loss": 2.9823,
+      "theoretical_loss": 4.179424652433255,
+      "tokens_seen": 280494080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045422735010402875,
+      "loss": 3.022,
+      "theoretical_loss": 4.177114959938745,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045399092112729335,
+      "loss": 3.0318,
+      "theoretical_loss": 4.174818977401182,
+      "tokens_seen": 283115520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000453754492150558,
+      "loss": 3.0489,
+      "theoretical_loss": 4.172536560531271,
+      "tokens_seen": 284426240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004535180631738226,
+      "loss": 3.0029,
+      "theoretical_loss": 4.170267567214273,
+      "tokens_seen": 285736960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004532816341970872,
+      "loss": 3.0065,
+      "theoretical_loss": 4.168011857467442,
+      "tokens_seen": 287047680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004530452052203518,
+      "loss": 3.0111,
+      "theoretical_loss": 4.1657692933984825,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045280877624361644,
+      "loss": 2.9825,
+      "theoretical_loss": 4.163539739164992,
+      "tokens_seen": 289669120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045257234726688104,
+      "loss": 3.0399,
+      "theoretical_loss": 4.161323060934883,
+      "tokens_seen": 290979840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045233591829014564,
+      "loss": 3.0392,
+      "theoretical_loss": 4.159119126847729,
+      "tokens_seen": 292290560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004520994893134103,
+      "loss": 3.0364,
+      "theoretical_loss": 4.1569278069770395,
+      "tokens_seen": 293601280
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045186306033667483,
+      "loss": 3.005,
+      "theoretical_loss": 4.154748973293404,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004516266313599395,
+      "loss": 3.0766,
+      "theoretical_loss": 4.152582499628515,
+      "tokens_seen": 296222720
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004513902023832041,
+      "loss": 3.0355,
+      "theoretical_loss": 4.150428261640019,
+      "tokens_seen": 297533440
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004511537734064687,
+      "loss": 3.044,
+      "theoretical_loss": 4.148286136777182,
+      "tokens_seen": 298844160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045091734442973333,
+      "loss": 3.0471,
+      "theoretical_loss": 4.1461560042473575,
+      "tokens_seen": 300154880
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045068091545299793,
+      "loss": 2.9748,
+      "theoretical_loss": 4.144037744983214,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004504444864762625,
+      "loss": 3.0411,
+      "theoretical_loss": 4.141931241610721,
+      "tokens_seen": 302776320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004502080574995271,
+      "loss": 3.0282,
+      "theoretical_loss": 4.139836378417861,
+      "tokens_seen": 304087040
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004499716285227918,
+      "loss": 2.9842,
+      "theoretical_loss": 4.1377530413240535,
+      "tokens_seen": 305397760
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044973519954605637,
+      "loss": 3.0131,
+      "theoretical_loss": 4.135681117850265,
+      "tokens_seen": 306708480
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044949877056932097,
+      "loss": 3.0458,
+      "theoretical_loss": 4.1336204970898,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004492623415925856,
+      "loss": 2.9775,
+      "theoretical_loss": 4.131571069679739,
+      "tokens_seen": 309329920
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044902591261585016,
+      "loss": 2.9762,
+      "theoretical_loss": 4.129532727773022,
+      "tokens_seen": 310640640
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004487894836391148,
+      "loss": 2.948,
+      "theoretical_loss": 4.127505365011135,
+      "tokens_seen": 311951360
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044855305466237947,
+      "loss": 2.9521,
+      "theoretical_loss": 4.125488876497428,
+      "tokens_seen": 313262080
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000448316625685644,
+      "loss": 2.9967,
+      "theoretical_loss": 4.123483158770997,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044808019670890866,
+      "loss": 2.9995,
+      "theoretical_loss": 4.121488109781147,
+      "tokens_seen": 315883520
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044784376773217326,
+      "loss": 2.9331,
+      "theoretical_loss": 4.11950362886242,
+      "tokens_seen": 317194240
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044760733875543786,
+      "loss": 2.9987,
+      "theoretical_loss": 4.1175296167101525,
+      "tokens_seen": 318504960
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004473709097787025,
+      "loss": 2.9851,
+      "theoretical_loss": 4.115565975356576,
+      "tokens_seen": 319815680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471344808019671,
+      "loss": 2.9513,
+      "theoretical_loss": 4.113612608147429,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004468980518252317,
+      "loss": 2.9103,
+      "theoretical_loss": 4.111669419719063,
+      "tokens_seen": 322437120
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004466616228484963,
+      "loss": 2.9268,
+      "theoretical_loss": 4.109736315976054,
+      "tokens_seen": 323747840
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044642519387176095,
+      "loss": 2.9573,
+      "theoretical_loss": 4.107813204069279,
+      "tokens_seen": 325058560
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004461887648950255,
+      "loss": 2.9633,
+      "theoretical_loss": 4.105899992374464,
+      "tokens_seen": 326369280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5416083918384781,
+      "debugging/distinct-1-grams": 0.7980827691008653,
+      "debugging/distinct-2-grams": 0.9666932256014988,
+      "debugging/entropy-1-grams": 6.182765900392966,
+      "debugging/entropy-2-grams": 7.172235247032516,
+      "debugging/length": 490.5,
+      "debugging/num_segments": 20,
+      "epoch": 0.12,
+      "objective/train/docs_used": 191985,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.0822439193725586,
+      "objective/train/theoretical_loss": 4.103996590471185,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.103996590471185,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044595233591829015,
+      "loss": 2.9667,
+      "theoretical_loss": 4.103996590471185,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004457159069415548,
+      "loss": 2.9485,
+      "theoretical_loss": 4.102102909122308,
+      "tokens_seen": 328990720
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044547947796481934,
+      "loss": 2.946,
+      "theoretical_loss": 4.1002188602538645,
+      "tokens_seen": 330301440
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000445243048988084,
+      "loss": 3.0404,
+      "theoretical_loss": 4.098344356935348,
+      "tokens_seen": 331612160
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004450066200113486,
+      "loss": 3.0199,
+      "theoretical_loss": 4.096479313360415,
+      "tokens_seen": 332922880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004447701910346132,
+      "loss": 2.9987,
+      "theoretical_loss": 4.094623644827995,
+      "tokens_seen": 334233600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044453376205787784,
+      "loss": 2.9626,
+      "theoretical_loss": 4.092777267723779,
+      "tokens_seen": 335544320
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044429733308114244,
+      "loss": 2.9576,
+      "theoretical_loss": 4.0909400995021,
+      "tokens_seen": 336855040
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044406090410440704,
+      "loss": 2.9351,
+      "theoretical_loss": 4.08911205866818,
+      "tokens_seen": 338165760
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044382447512767163,
+      "loss": 3.0207,
+      "theoretical_loss": 4.087293064760739,
+      "tokens_seen": 339476480
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435880461509363,
+      "loss": 2.9825,
+      "theoretical_loss": 4.0854830383349565,
+      "tokens_seen": 340787200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004433516171742009,
+      "loss": 2.9368,
+      "theoretical_loss": 4.083681900945785,
+      "tokens_seen": 342097920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004431151881974655,
+      "loss": 2.9302,
+      "theoretical_loss": 4.081889575131589,
+      "tokens_seen": 343408640
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044287875922073013,
+      "loss": 2.9436,
+      "theoretical_loss": 4.080105984398123,
+      "tokens_seen": 344719360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004426423302439947,
+      "loss": 2.9709,
+      "theoretical_loss": 4.078331053202827,
+      "tokens_seen": 346030080
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004424059012672593,
+      "loss": 2.8916,
+      "theoretical_loss": 4.076564706939434,
+      "tokens_seen": 347340800
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000442169472290524,
+      "loss": 2.9711,
+      "theoretical_loss": 4.074806871922881,
+      "tokens_seen": 348651520
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004419330433137885,
+      "loss": 2.9344,
+      "theoretical_loss": 4.0730574753745294,
+      "tokens_seen": 349962240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004416966143370532,
+      "loss": 2.946,
+      "theoretical_loss": 4.071316445407663,
+      "tokens_seen": 351272960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044146018536031777,
+      "loss": 2.9271,
+      "theoretical_loss": 4.06958371101328,
+      "tokens_seen": 352583680
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044122375638358237,
+      "loss": 2.9213,
+      "theoretical_loss": 4.067859202046162,
+      "tokens_seen": 353894400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000440987327406847,
+      "loss": 2.9237,
+      "theoretical_loss": 4.066142849211214,
+      "tokens_seen": 355205120
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004407508984301116,
+      "loss": 2.9702,
+      "theoretical_loss": 4.064434584050067,
+      "tokens_seen": 356515840
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004405144694533762,
+      "loss": 2.9979,
+      "theoretical_loss": 4.062734338927946,
+      "tokens_seen": 357826560
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004402780404766408,
+      "loss": 2.9666,
+      "theoretical_loss": 4.061042047020785,
+      "tokens_seen": 359137280
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044004161149990546,
+      "loss": 2.9475,
+      "theoretical_loss": 4.0593576423025945,
+      "tokens_seen": 360448000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043980518252317,
+      "loss": 2.9903,
+      "theoretical_loss": 4.057681059533071,
+      "tokens_seen": 361758720
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043956875354643466,
+      "loss": 3.0275,
+      "theoretical_loss": 4.056012234245436,
+      "tokens_seen": 363069440
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004393323245696993,
+      "loss": 2.9532,
+      "theoretical_loss": 4.05435110273452,
+      "tokens_seen": 364380160
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043909589559296385,
+      "loss": 3.0061,
+      "theoretical_loss": 4.052697602045054,
+      "tokens_seen": 365690880
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388594666162285,
+      "loss": 2.9492,
+      "theoretical_loss": 4.0510516699602,
+      "tokens_seen": 367001600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004386230376394931,
+      "loss": 2.9299,
+      "theoretical_loss": 4.049413244990284,
+      "tokens_seen": 368312320
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004383866086627577,
+      "loss": 2.917,
+      "theoretical_loss": 4.04778226636175,
+      "tokens_seen": 369623040
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043815017968602235,
+      "loss": 2.9734,
+      "theoretical_loss": 4.046158674006312,
+      "tokens_seen": 370933760
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043791375070928695,
+      "loss": 2.9703,
+      "theoretical_loss": 4.044542408550312,
+      "tokens_seen": 372244480
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043767732173255155,
+      "loss": 2.9203,
+      "theoretical_loss": 4.042933411304277,
+      "tokens_seen": 373555200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043744089275581614,
+      "loss": 2.9796,
+      "theoretical_loss": 4.041331624252669,
+      "tokens_seen": 374865920
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004372044637790808,
+      "loss": 3.0013,
+      "theoretical_loss": 4.039736990043815,
+      "tokens_seen": 376176640
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004369680348023454,
+      "loss": 2.9045,
+      "theoretical_loss": 4.038149451980039,
+      "tokens_seen": 377487360
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043673160582561,
+      "loss": 2.8935,
+      "theoretical_loss": 4.0365689540079535,
+      "tokens_seen": 378798080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043649517684887464,
+      "loss": 2.9313,
+      "theoretical_loss": 4.034995440708947,
+      "tokens_seen": 380108800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004362587478721392,
+      "loss": 2.9622,
+      "theoretical_loss": 4.033428857289831,
+      "tokens_seen": 381419520
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043602231889540384,
+      "loss": 2.9375,
+      "theoretical_loss": 4.031869149573668,
+      "tokens_seen": 382730240
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004357858899186685,
+      "loss": 2.8995,
+      "theoretical_loss": 4.030316263990749,
+      "tokens_seen": 384040960
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043554946094193303,
+      "loss": 2.9652,
+      "theoretical_loss": 4.028770147569752,
+      "tokens_seen": 385351680
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004353130319651977,
+      "loss": 2.8495,
+      "theoretical_loss": 4.0272307479290435,
+      "tokens_seen": 386662400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004350766029884623,
+      "loss": 2.9227,
+      "theoretical_loss": 4.025698013268144,
+      "tokens_seen": 387973120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004348401740117269,
+      "loss": 2.9515,
+      "theoretical_loss": 4.024171892359345,
+      "tokens_seen": 389283840
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043460374503499153,
+      "loss": 2.9514,
+      "theoretical_loss": 4.0226523345394645,
+      "tokens_seen": 390594560
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043436731605825613,
+      "loss": 2.9178,
+      "theoretical_loss": 4.021139289701768,
+      "tokens_seen": 391905280
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 228129,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7468600273132324,
+      "objective/train/theoretical_loss": 4.019632708288012,
+      "objective/train/tokens_used": 413676000,
+      "theoretical_loss": 4.019632708288012,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004341308870815207,
+      "loss": 2.9228,
+      "theoretical_loss": 4.019632708288012,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004338944581047853,
+      "loss": 2.9153,
+      "theoretical_loss": 4.018132541280638,
+      "tokens_seen": 394526720
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043365802912805,
+      "loss": 2.9176,
+      "theoretical_loss": 4.016638740195099,
+      "tokens_seen": 395837440
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004334216001513145,
+      "loss": 2.9225,
+      "theoretical_loss": 4.015151257072327,
+      "tokens_seen": 397148160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043318517117457917,
+      "loss": 2.9261,
+      "theoretical_loss": 4.013670044471318,
+      "tokens_seen": 398458880
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004329487421978438,
+      "loss": 2.9174,
+      "theoretical_loss": 4.012195055461859,
+      "tokens_seen": 399769600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043271231322110837,
+      "loss": 2.9585,
+      "theoretical_loss": 4.010726243617374,
+      "tokens_seen": 401080320
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000432475884244373,
+      "loss": 2.9271,
+      "theoretical_loss": 4.009263563007895,
+      "tokens_seen": 402391040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004322394552676376,
+      "loss": 2.9148,
+      "theoretical_loss": 4.007806968193149,
+      "tokens_seen": 403701760
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004320030262909022,
+      "loss": 2.9607,
+      "theoretical_loss": 4.006356414215775,
+      "tokens_seen": 405012480
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043176659731416686,
+      "loss": 2.9537,
+      "theoretical_loss": 4.004911856594639,
+      "tokens_seen": 406323200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004315301683374314,
+      "loss": 2.9175,
+      "theoretical_loss": 4.00347325131828,
+      "tokens_seen": 407633920
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043129373936069606,
+      "loss": 2.9827,
+      "theoretical_loss": 4.002040554838458,
+      "tokens_seen": 408944640
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043105731038396066,
+      "loss": 2.926,
+      "theoretical_loss": 4.000613724063812,
+      "tokens_seen": 410255360
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043082088140722525,
+      "loss": 2.884,
+      "theoretical_loss": 3.9991927163536243,
+      "tokens_seen": 411566080
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004305844524304899,
+      "loss": 2.9218,
+      "theoretical_loss": 3.997777489511691,
+      "tokens_seen": 412876800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004303480234537545,
+      "loss": 2.9585,
+      "theoretical_loss": 3.9963680017802945,
+      "tokens_seen": 414187520
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004301115944770191,
+      "loss": 2.9845,
+      "theoretical_loss": 3.9949642118342794,
+      "tokens_seen": 415498240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004298751655002837,
+      "loss": 3.0169,
+      "theoretical_loss": 3.993566078775218,
+      "tokens_seen": 416808960
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042963873652354835,
+      "loss": 2.9619,
+      "theoretical_loss": 3.9921735621256835,
+      "tokens_seen": 418119680
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042940230754681295,
+      "loss": 2.9916,
+      "theoretical_loss": 3.990786621823611,
+      "tokens_seen": 419430400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042916587857007754,
+      "loss": 3.0605,
+      "theoretical_loss": 3.9894052182167536,
+      "tokens_seen": 420741120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004289294495933422,
+      "loss": 2.9475,
+      "theoretical_loss": 3.988029312057227,
+      "tokens_seen": 422051840
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042869302061660674,
+      "loss": 2.9815,
+      "theoretical_loss": 3.986658864496149,
+      "tokens_seen": 423362560
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004284565916398714,
+      "loss": 2.9608,
+      "theoretical_loss": 3.985293837078359,
+      "tokens_seen": 424673280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000428220162663136,
+      "loss": 2.943,
+      "theoretical_loss": 3.9839341917372293,
+      "tokens_seen": 425984000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004279837336864006,
+      "loss": 3.0196,
+      "theoretical_loss": 3.9825798907895598,
+      "tokens_seen": 427294720
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042774730470966524,
+      "loss": 2.9766,
+      "theoretical_loss": 3.9812308969305508,
+      "tokens_seen": 428605440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042751087573292984,
+      "loss": 3.0151,
+      "theoretical_loss": 3.9798871732288617,
+      "tokens_seen": 429916160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042727444675619443,
+      "loss": 2.9232,
+      "theoretical_loss": 3.9785486831217467,
+      "tokens_seen": 431226880
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042703801777945903,
+      "loss": 2.9393,
+      "theoretical_loss": 3.9772153904102705,
+      "tokens_seen": 432537600
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004268015888027237,
+      "loss": 2.9605,
+      "theoretical_loss": 3.9758872592545957,
+      "tokens_seen": 433848320
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004265651598259883,
+      "loss": 2.9803,
+      "theoretical_loss": 3.974564254169353,
+      "tokens_seen": 435159040
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004263287308492529,
+      "loss": 2.978,
+      "theoretical_loss": 3.9732463400190756,
+      "tokens_seen": 436469760
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042609230187251753,
+      "loss": 2.9034,
+      "theoretical_loss": 3.9719334820137124,
+      "tokens_seen": 437780480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042585587289578207,
+      "loss": 2.9457,
+      "theoretical_loss": 3.9706256457042133,
+      "tokens_seen": 439091200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004256194439190467,
+      "loss": 2.9394,
+      "theoretical_loss": 3.969322796978175,
+      "tokens_seen": 440401920
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004253830149423114,
+      "loss": 2.9263,
+      "theoretical_loss": 3.968024902055565,
+      "tokens_seen": 441712640
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004251465859655759,
+      "loss": 2.927,
+      "theoretical_loss": 3.966731927484508,
+      "tokens_seen": 443023360
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042491015698884057,
+      "loss": 2.9259,
+      "theoretical_loss": 3.9654438401371372,
+      "tokens_seen": 444334080
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042467372801210517,
+      "loss": 2.9706,
+      "theoretical_loss": 3.9641606072055113,
+      "tokens_seen": 445644800
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042443729903536976,
+      "loss": 2.9153,
+      "theoretical_loss": 3.9628821961975955,
+      "tokens_seen": 446955520
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004242008700586344,
+      "loss": 2.9132,
+      "theoretical_loss": 3.9616085749333028,
+      "tokens_seen": 448266240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000423964441081899,
+      "loss": 2.9324,
+      "theoretical_loss": 3.9603397115406005,
+      "tokens_seen": 449576960
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004237280121051636,
+      "loss": 2.9444,
+      "theoretical_loss": 3.9590755744516684,
+      "tokens_seen": 450887680
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004234915831284282,
+      "loss": 2.9114,
+      "theoretical_loss": 3.9578161323991243,
+      "tokens_seen": 452198400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042325515415169286,
+      "loss": 2.9374,
+      "theoretical_loss": 3.9565613544123046,
+      "tokens_seen": 453509120
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004230187251749574,
+      "loss": 2.8609,
+      "theoretical_loss": 3.955311209813596,
+      "tokens_seen": 454819840
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042278229619822206,
+      "loss": 2.8826,
+      "theoretical_loss": 3.9540656682148327,
+      "tokens_seen": 456130560
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004225458672214867,
+      "loss": 2.9145,
+      "theoretical_loss": 3.952824699513741,
+      "tokens_seen": 457441280
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 262792,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.892580032348633,
+      "objective/train/theoretical_loss": 3.951588273890438,
+      "objective/train/tokens_used": 479212000,
+      "theoretical_loss": 3.951588273890438,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042230943824475125,
+      "loss": 2.9221,
+      "theoretical_loss": 3.951588273890438,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004220730092680159,
+      "loss": 2.9077,
+      "theoretical_loss": 3.950356361803988,
+      "tokens_seen": 460062720
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004218365802912805,
+      "loss": 2.9355,
+      "theoretical_loss": 3.949128933989004,
+      "tokens_seen": 461373440
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004216001513145451,
+      "loss": 2.9047,
+      "theoretical_loss": 3.9479059614523058,
+      "tokens_seen": 462684160
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042136372233780975,
+      "loss": 2.8407,
+      "theoretical_loss": 3.9466874154696234,
+      "tokens_seen": 463994880
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042112729336107435,
+      "loss": 2.8673,
+      "theoretical_loss": 3.945473267582355,
+      "tokens_seen": 465305600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042089086438433894,
+      "loss": 2.835,
+      "theoretical_loss": 3.9442634895943662,
+      "tokens_seen": 466616320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042065443540760354,
+      "loss": 2.8983,
+      "theoretical_loss": 3.943058053568841,
+      "tokens_seen": 467927040
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004204180064308682,
+      "loss": 2.875,
+      "theoretical_loss": 3.94185693182518,
+      "tokens_seen": 469237760
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004201815774541328,
+      "loss": 2.8836,
+      "theoretical_loss": 3.9406600969359413,
+      "tokens_seen": 470548480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199451484773974,
+      "loss": 2.8861,
+      "theoretical_loss": 3.9394675217238277,
+      "tokens_seen": 471859200
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041970871950066204,
+      "loss": 2.8541,
+      "theoretical_loss": 3.9382791792587195,
+      "tokens_seen": 473169920
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004194722905239266,
+      "loss": 2.9094,
+      "theoretical_loss": 3.9370950428547475,
+      "tokens_seen": 474480640
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041923586154719123,
+      "loss": 2.8365,
+      "theoretical_loss": 3.9359150860674106,
+      "tokens_seen": 475791360
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004189994325704559,
+      "loss": 2.8452,
+      "theoretical_loss": 3.9347392826907353,
+      "tokens_seen": 477102080
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041876300359372043,
+      "loss": 2.8346,
+      "theoretical_loss": 3.933567606754476,
+      "tokens_seen": 478412800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004185265746169851,
+      "loss": 2.8767,
+      "theoretical_loss": 3.9324000325213557,
+      "tokens_seen": 479723520
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004182901456402497,
+      "loss": 2.8949,
+      "theoretical_loss": 3.931236534484346,
+      "tokens_seen": 481034240
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004180537166635143,
+      "loss": 2.8451,
+      "theoretical_loss": 3.930077087363987,
+      "tokens_seen": 482344960
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041781728768677893,
+      "loss": 2.8033,
+      "theoretical_loss": 3.928921666105744,
+      "tokens_seen": 483655680
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004175808587100435,
+      "loss": 2.8546,
+      "theoretical_loss": 3.9277702458774058,
+      "tokens_seen": 484966400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004173444297333081,
+      "loss": 2.8101,
+      "theoretical_loss": 3.926622802066514,
+      "tokens_seen": 486277120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004171080007565727,
+      "loss": 2.8595,
+      "theoretical_loss": 3.925479310277835,
+      "tokens_seen": 487587840
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041687157177983737,
+      "loss": 2.928,
+      "theoretical_loss": 3.924339746330864,
+      "tokens_seen": 488898560
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004166351428031019,
+      "loss": 2.8558,
+      "theoretical_loss": 3.923204086257365,
+      "tokens_seen": 490209280
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041639871382636657,
+      "loss": 2.8963,
+      "theoretical_loss": 3.9220723062989444,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004161622848496312,
+      "loss": 2.9296,
+      "theoretical_loss": 3.920944382904665,
+      "tokens_seen": 492830720
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041592585587289576,
+      "loss": 2.9416,
+      "theoretical_loss": 3.919820292728682,
+      "tokens_seen": 494141440
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004156894268961604,
+      "loss": 2.8974,
+      "theoretical_loss": 3.918700012627923,
+      "tokens_seen": 495452160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000415452997919425,
+      "loss": 2.911,
+      "theoretical_loss": 3.9175835196597957,
+      "tokens_seen": 496762880
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004152165689426896,
+      "loss": 2.8654,
+      "theoretical_loss": 3.9164707910799264,
+      "tokens_seen": 498073600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041498013996595426,
+      "loss": 2.8717,
+      "theoretical_loss": 3.9153618043399296,
+      "tokens_seen": 499384320
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041474371098921886,
+      "loss": 2.8459,
+      "theoretical_loss": 3.9142565370852136,
+      "tokens_seen": 500695040
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041450728201248346,
+      "loss": 2.8187,
+      "theoretical_loss": 3.9131549671528116,
+      "tokens_seen": 502005760
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041427085303574805,
+      "loss": 2.8665,
+      "theoretical_loss": 3.912057072569243,
+      "tokens_seen": 503316480
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004140344240590127,
+      "loss": 2.8217,
+      "theoretical_loss": 3.910962831548404,
+      "tokens_seen": 504627200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004137979950822773,
+      "loss": 2.8274,
+      "theoretical_loss": 3.909872222489491,
+      "tokens_seen": 505937920
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004135615661055419,
+      "loss": 2.845,
+      "theoretical_loss": 3.908785223974948,
+      "tokens_seen": 507248640
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041332513712880655,
+      "loss": 2.8363,
+      "theoretical_loss": 3.9077018147684415,
+      "tokens_seen": 508559360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004130887081520711,
+      "loss": 2.878,
+      "theoretical_loss": 3.906621973812867,
+      "tokens_seen": 509870080
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041285227917533575,
+      "loss": 2.8849,
+      "theoretical_loss": 3.9055456802283812,
+      "tokens_seen": 511180800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004126158501986004,
+      "loss": 2.8734,
+      "theoretical_loss": 3.904472913310456,
+      "tokens_seen": 512491520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041237942122186494,
+      "loss": 2.7937,
+      "theoretical_loss": 3.9034036525279645,
+      "tokens_seen": 513802240
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004121429922451296,
+      "loss": 2.8423,
+      "theoretical_loss": 3.902337877521294,
+      "tokens_seen": 515112960
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004119065632683942,
+      "loss": 2.7962,
+      "theoretical_loss": 3.9012755681004756,
+      "tokens_seen": 516423680
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004116701342916588,
+      "loss": 2.821,
+      "theoretical_loss": 3.9002167042433475,
+      "tokens_seen": 517734400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041143370531492344,
+      "loss": 2.8321,
+      "theoretical_loss": 3.899161266093741,
+      "tokens_seen": 519045120
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041119727633818804,
+      "loss": 2.8604,
+      "theoretical_loss": 3.8981092339596852,
+      "tokens_seen": 520355840
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041096084736145263,
+      "loss": 2.8132,
+      "theoretical_loss": 3.8970605883116436,
+      "tokens_seen": 521666560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041072441838471723,
+      "loss": 2.8032,
+      "theoretical_loss": 3.896015309780767,
+      "tokens_seen": 522977280
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 298912,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.98516845703125,
+      "objective/train/theoretical_loss": 3.8949733791571752,
+      "objective/train/tokens_used": 544748000,
+      "theoretical_loss": 3.8949733791571752,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004104879894079819,
+      "loss": 2.7451,
+      "theoretical_loss": 3.8949733791571752,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004102515604312464,
+      "loss": 2.8263,
+      "theoretical_loss": 3.8939347773882584,
+      "tokens_seen": 525598720
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004100151314545111,
+      "loss": 2.8216,
+      "theoretical_loss": 3.892899485576998,
+      "tokens_seen": 526909440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040977870247777573,
+      "loss": 2.8367,
+      "theoretical_loss": 3.8918674849803176,
+      "tokens_seen": 528220160
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004095422735010403,
+      "loss": 2.826,
+      "theoretical_loss": 3.890838757007447,
+      "tokens_seen": 529530880
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004093058445243049,
+      "loss": 2.7991,
+      "theoretical_loss": 3.889813283218313,
+      "tokens_seen": 530841600
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004090694155475695,
+      "loss": 2.8456,
+      "theoretical_loss": 3.888791045321949,
+      "tokens_seen": 532152320
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004088329865708341,
+      "loss": 2.8602,
+      "theoretical_loss": 3.887772025174925,
+      "tokens_seen": 533463040
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040859655759409877,
+      "loss": 2.8292,
+      "theoretical_loss": 3.8867562047797986,
+      "tokens_seen": 534773760
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040836012861736337,
+      "loss": 2.8996,
+      "theoretical_loss": 3.885743566283587,
+      "tokens_seen": 536084480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040812369964062797,
+      "loss": 2.8527,
+      "theoretical_loss": 3.8847340919762576,
+      "tokens_seen": 537395200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040788727066389256,
+      "loss": 2.8743,
+      "theoretical_loss": 3.883727764289236,
+      "tokens_seen": 538705920
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040765084168715716,
+      "loss": 2.8987,
+      "theoretical_loss": 3.8827245657939375,
+      "tokens_seen": 540016640
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004074144127104218,
+      "loss": 2.8296,
+      "theoretical_loss": 3.8817244792003143,
+      "tokens_seen": 541327360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004071779837336864,
+      "loss": 2.8508,
+      "theoretical_loss": 3.8807274873554216,
+      "tokens_seen": 542638080
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.000406941554756951,
+      "loss": 2.8473,
+      "theoretical_loss": 3.8797335732420035,
+      "tokens_seen": 543948800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004067051257802156,
+      "loss": 2.82,
+      "theoretical_loss": 3.878742719977094,
+      "tokens_seen": 545259520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040646869680348026,
+      "loss": 2.839,
+      "theoretical_loss": 3.877754910810639,
+      "tokens_seen": 546570240
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040623226782674485,
+      "loss": 2.8439,
+      "theoretical_loss": 3.876770129124133,
+      "tokens_seen": 547880960
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040599583885000945,
+      "loss": 2.8545,
+      "theoretical_loss": 3.8757883584292774,
+      "tokens_seen": 549191680
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004057594098732741,
+      "loss": 2.8427,
+      "theoretical_loss": 3.8748095823666473,
+      "tokens_seen": 550502400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040552298089653865,
+      "loss": 2.8971,
+      "theoretical_loss": 3.8738337847043836,
+      "tokens_seen": 551813120
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004052865519198033,
+      "loss": 2.8655,
+      "theoretical_loss": 3.8728609493368986,
+      "tokens_seen": 553123840
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004050501229430679,
+      "loss": 2.8534,
+      "theoretical_loss": 3.8718910602835943,
+      "tokens_seen": 554434560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004048136939663325,
+      "loss": 2.8304,
+      "theoretical_loss": 3.870924101687603,
+      "tokens_seen": 555745280
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040457726498959715,
+      "loss": 2.8699,
+      "theoretical_loss": 3.869960057814537,
+      "tokens_seen": 557056000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040434083601286174,
+      "loss": 2.8513,
+      "theoretical_loss": 3.868998913051261,
+      "tokens_seen": 558366720
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040410440703612634,
+      "loss": 2.882,
+      "theoretical_loss": 3.868040651904671,
+      "tokens_seen": 559677440
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040386797805939094,
+      "loss": 2.8468,
+      "theoretical_loss": 3.867085259000496,
+      "tokens_seen": 560988160
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004036315490826556,
+      "loss": 2.8575,
+      "theoretical_loss": 3.8661327190821124,
+      "tokens_seen": 562298880
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004033951201059202,
+      "loss": 2.8521,
+      "theoretical_loss": 3.8651830170093673,
+      "tokens_seen": 563609600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004031586911291848,
+      "loss": 2.8827,
+      "theoretical_loss": 3.864236137757427,
+      "tokens_seen": 564920320
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040292226215244944,
+      "loss": 2.8591,
+      "theoretical_loss": 3.863292066415628,
+      "tokens_seen": 566231040
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.000402685833175714,
+      "loss": 2.9049,
+      "theoretical_loss": 3.862350788186353,
+      "tokens_seen": 567541760
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040244940419897863,
+      "loss": 2.8928,
+      "theoretical_loss": 3.8614122883839093,
+      "tokens_seen": 568852480
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004022129752222433,
+      "loss": 2.9413,
+      "theoretical_loss": 3.8604765524334312,
+      "tokens_seen": 570163200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004019765462455078,
+      "loss": 2.9571,
+      "theoretical_loss": 3.8595435658697888,
+      "tokens_seen": 571473920
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004017401172687725,
+      "loss": 2.9104,
+      "theoretical_loss": 3.858613314336514,
+      "tokens_seen": 572784640
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004015036882920371,
+      "loss": 2.9053,
+      "theoretical_loss": 3.8576857835847367,
+      "tokens_seen": 574095360
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040126725931530167,
+      "loss": 2.9107,
+      "theoretical_loss": 3.8567609594721333,
+      "tokens_seen": 575406080
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004010308303385663,
+      "loss": 2.8914,
+      "theoretical_loss": 3.855838827961894,
+      "tokens_seen": 576716800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004007944013618309,
+      "loss": 2.8867,
+      "theoretical_loss": 3.8549193751216935,
+      "tokens_seen": 578027520
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004005579723850955,
+      "loss": 2.955,
+      "theoretical_loss": 3.854002587122681,
+      "tokens_seen": 579338240
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004003215434083601,
+      "loss": 2.8915,
+      "theoretical_loss": 3.853088450238478,
+      "tokens_seen": 580648960
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040008511443162477,
+      "loss": 2.836,
+      "theoretical_loss": 3.8521769508441905,
+      "tokens_seen": 581959680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003998486854548893,
+      "loss": 2.8512,
+      "theoretical_loss": 3.8512680754154323,
+      "tokens_seen": 583270400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039961225647815396,
+      "loss": 2.8983,
+      "theoretical_loss": 3.850361810527361,
+      "tokens_seen": 584581120
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003993758275014186,
+      "loss": 2.8751,
+      "theoretical_loss": 3.8494581428537202,
+      "tokens_seen": 585891840
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039913939852468316,
+      "loss": 2.8825,
+      "theoretical_loss": 3.8485570591659006,
+      "tokens_seen": 587202560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003989029695479478,
+      "loss": 2.8639,
+      "theoretical_loss": 3.847658546332007,
+      "tokens_seen": 588513280
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 333293,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.010171413421631,
+      "objective/train/theoretical_loss": 3.8467625913159385,
+      "objective/train/tokens_used": 610284000,
+      "theoretical_loss": 3.8467625913159385,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003986665405712124,
+      "loss": 2.8745,
+      "theoretical_loss": 3.8467625913159385,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.000398430111594477,
+      "loss": 2.9131,
+      "theoretical_loss": 3.845869181176477,
+      "tokens_seen": 591134720
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039819368261774166,
+      "loss": 2.8483,
+      "theoretical_loss": 3.844978303066389,
+      "tokens_seen": 592445440
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039795725364100625,
+      "loss": 2.8512,
+      "theoretical_loss": 3.8440899442315395,
+      "tokens_seen": 593756160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039772082466427085,
+      "loss": 2.8047,
+      "theoretical_loss": 3.843204092010007,
+      "tokens_seen": 595066880
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039748439568753545,
+      "loss": 2.7938,
+      "theoretical_loss": 3.8423207338312233,
+      "tokens_seen": 596377600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003972479667108001,
+      "loss": 2.8365,
+      "theoretical_loss": 3.841439857215109,
+      "tokens_seen": 597688320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003970115377340647,
+      "loss": 2.7964,
+      "theoretical_loss": 3.84056144977123,
+      "tokens_seen": 598999040
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003967751087573293,
+      "loss": 2.8111,
+      "theoretical_loss": 3.839685499197956,
+      "tokens_seen": 600309760
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039653867978059395,
+      "loss": 2.7922,
+      "theoretical_loss": 3.8388119932816327,
+      "tokens_seen": 601620480
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003963022508038585,
+      "loss": 2.8055,
+      "theoretical_loss": 3.8379409198957637,
+      "tokens_seen": 602931200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039606582182712314,
+      "loss": 2.813,
+      "theoretical_loss": 3.837072267000199,
+      "tokens_seen": 604241920
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003958293928503878,
+      "loss": 2.8134,
+      "theoretical_loss": 3.8362060226403347,
+      "tokens_seen": 605552640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039559296387365234,
+      "loss": 2.8747,
+      "theoretical_loss": 3.835342174946324,
+      "tokens_seen": 606863360
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.000395356534896917,
+      "loss": 2.8115,
+      "theoretical_loss": 3.834480712132291,
+      "tokens_seen": 608174080
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003951201059201816,
+      "loss": 2.8253,
+      "theoretical_loss": 3.833621622495561,
+      "tokens_seen": 609484800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003948836769434462,
+      "loss": 2.8841,
+      "theoretical_loss": 3.8327648944158907,
+      "tokens_seen": 610795520
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039464724796671084,
+      "loss": 2.8109,
+      "theoretical_loss": 3.8319105163547187,
+      "tokens_seen": 612106240
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039441081898997543,
+      "loss": 2.8429,
+      "theoretical_loss": 3.8310584768544125,
+      "tokens_seen": 613416960
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039417439001324003,
+      "loss": 2.8154,
+      "theoretical_loss": 3.8302087645375327,
+      "tokens_seen": 614727680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039393796103650463,
+      "loss": 2.8656,
+      "theoretical_loss": 3.829361368106099,
+      "tokens_seen": 616038400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003937015320597693,
+      "loss": 2.7981,
+      "theoretical_loss": 3.8285162763408707,
+      "tokens_seen": 617349120
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003934651030830338,
+      "loss": 2.7823,
+      "theoretical_loss": 3.827673478100631,
+      "tokens_seen": 618659840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003932286741062985,
+      "loss": 2.7954,
+      "theoretical_loss": 3.8268329623214803,
+      "tokens_seen": 619970560
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003929922451295631,
+      "loss": 2.8648,
+      "theoretical_loss": 3.8259947180161373,
+      "tokens_seen": 621281280
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039275581615282767,
+      "loss": 2.8326,
+      "theoretical_loss": 3.8251587342732485,
+      "tokens_seen": 622592000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003925193871760923,
+      "loss": 2.8281,
+      "theoretical_loss": 3.824325000256705,
+      "tokens_seen": 623902720
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003922829581993569,
+      "loss": 2.7531,
+      "theoretical_loss": 3.8234935052049663,
+      "tokens_seen": 625213440
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003920465292226215,
+      "loss": 2.7447,
+      "theoretical_loss": 3.8226642384303933,
+      "tokens_seen": 626524160
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039181010024588617,
+      "loss": 2.8173,
+      "theoretical_loss": 3.821837189318586,
+      "tokens_seen": 627834880
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039157367126915077,
+      "loss": 2.7845,
+      "theoretical_loss": 3.82101234732773,
+      "tokens_seen": 629145600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039133724229241536,
+      "loss": 2.8092,
+      "theoretical_loss": 3.820189701987953,
+      "tokens_seen": 630456320
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039110081331567996,
+      "loss": 2.8167,
+      "theoretical_loss": 3.819369242900682,
+      "tokens_seen": 631767040
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003908643843389446,
+      "loss": 2.8124,
+      "theoretical_loss": 3.8185509597380105,
+      "tokens_seen": 633077760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003906279553622092,
+      "loss": 2.8588,
+      "theoretical_loss": 3.8177348422420785,
+      "tokens_seen": 634388480
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003903915263854738,
+      "loss": 2.8766,
+      "theoretical_loss": 3.8169208802244476,
+      "tokens_seen": 635699200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039015509740873846,
+      "loss": 2.864,
+      "theoretical_loss": 3.8161090635654937,
+      "tokens_seen": 637009920
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000389918668432003,
+      "loss": 2.8529,
+      "theoretical_loss": 3.815299382213799,
+      "tokens_seen": 638320640
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038968223945526765,
+      "loss": 2.8858,
+      "theoretical_loss": 3.8144918261855536,
+      "tokens_seen": 639631360
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003894458104785323,
+      "loss": 2.8387,
+      "theoretical_loss": 3.813686385563966,
+      "tokens_seen": 640942080
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038920938150179685,
+      "loss": 2.8272,
+      "theoretical_loss": 3.812883050498675,
+      "tokens_seen": 642252800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003889729525250615,
+      "loss": 2.9091,
+      "theoretical_loss": 3.812081811205168,
+      "tokens_seen": 643563520
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003887365235483261,
+      "loss": 2.8777,
+      "theoretical_loss": 3.811282657964214,
+      "tokens_seen": 644874240
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003885000945715907,
+      "loss": 2.8015,
+      "theoretical_loss": 3.8104855811212905,
+      "tokens_seen": 646184960
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038826366559485535,
+      "loss": 2.8613,
+      "theoretical_loss": 3.8096905710860254,
+      "tokens_seen": 647495680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038802723661811994,
+      "loss": 2.8904,
+      "theoretical_loss": 3.808897618331641,
+      "tokens_seen": 648806400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038779080764138454,
+      "loss": 2.8777,
+      "theoretical_loss": 3.8081067133944053,
+      "tokens_seen": 650117120
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038755437866464914,
+      "loss": 2.8393,
+      "theoretical_loss": 3.8073178468730884,
+      "tokens_seen": 651427840
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003873179496879138,
+      "loss": 2.8258,
+      "theoretical_loss": 3.806531009428425,
+      "tokens_seen": 652738560
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038708152071117833,
+      "loss": 2.8412,
+      "theoretical_loss": 3.805746191782582,
+      "tokens_seen": 654049280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4831024939875183,
+      "debugging/distinct-1-grams": 0.8154550796195709,
+      "debugging/distinct-2-grams": 0.9726187101634798,
+      "debugging/entropy-1-grams": 5.88298844007536,
+      "debugging/entropy-2-grams": 6.629837829827698,
+      "debugging/length": 534.5,
+      "debugging/num_segments": 10,
+      "epoch": 0.23,
+      "objective/train/docs_used": 369378,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.009995222091675,
+      "objective/train/theoretical_loss": 3.8049633847186337,
+      "objective/train/tokens_used": 675820000,
+      "theoretical_loss": 3.8049633847186337,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000386845091734443,
+      "loss": 2.8184,
+      "theoretical_loss": 3.8049633847186337,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038660866275770764,
+      "loss": 2.7351,
+      "theoretical_loss": 3.80418257908004,
+      "tokens_seen": 656670720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003863722337809722,
+      "loss": 2.8029,
+      "theoretical_loss": 3.803403765770131,
+      "tokens_seen": 657981440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038613580480423683,
+      "loss": 2.7912,
+      "theoretical_loss": 3.802626935751598,
+      "tokens_seen": 659292160
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038589937582750143,
+      "loss": 2.7949,
+      "theoretical_loss": 3.801852080045988,
+      "tokens_seen": 660602880
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038566294685076603,
+      "loss": 2.7934,
+      "theoretical_loss": 3.801079189733206,
+      "tokens_seen": 661913600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003854265178740307,
+      "loss": 2.8019,
+      "theoretical_loss": 3.8003082559510197,
+      "tokens_seen": 663224320
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003851900888972953,
+      "loss": 2.8398,
+      "theoretical_loss": 3.799539269894571,
+      "tokens_seen": 664535040
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003849536599205599,
+      "loss": 2.8581,
+      "theoretical_loss": 3.798772222815893,
+      "tokens_seen": 665845760
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038471723094382447,
+      "loss": 2.7848,
+      "theoretical_loss": 3.79800710602343,
+      "tokens_seen": 667156480
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003844926234159259,
+      "loss": 2.8385,
+      "theoretical_loss": 3.7972439108815657,
+      "tokens_seen": 668467200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038425619443919047,
+      "loss": 2.8834,
+      "theoretical_loss": 3.7964826288101508,
+      "tokens_seen": 669777920
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003840315869112919,
+      "loss": 2.8737,
+      "theoretical_loss": 3.795723251284044,
+      "tokens_seen": 671088640
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038379515793455647,
+      "loss": 2.9181,
+      "theoretical_loss": 3.7949657698326487,
+      "tokens_seen": 672399360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038355872895782107,
+      "loss": 2.9011,
+      "theoretical_loss": 3.79421017603946,
+      "tokens_seen": 673710080
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833222999810857,
+      "loss": 2.852,
+      "theoretical_loss": 3.7934564615416146,
+      "tokens_seen": 675020800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003830858710043503,
+      "loss": 2.8606,
+      "theoretical_loss": 3.7927046180294455,
+      "tokens_seen": 676331520
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003828494420276149,
+      "loss": 2.8736,
+      "theoretical_loss": 3.7919546372460418,
+      "tokens_seen": 677642240
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003826130130508795,
+      "loss": 2.8608,
+      "theoretical_loss": 3.7912065109868127,
+      "tokens_seen": 678952960
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038237658407414417,
+      "loss": 2.8368,
+      "theoretical_loss": 3.790460231099053,
+      "tokens_seen": 680263680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038214015509740876,
+      "loss": 2.892,
+      "theoretical_loss": 3.789715789481521,
+      "tokens_seen": 681574400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038190372612067336,
+      "loss": 2.8421,
+      "theoretical_loss": 3.788973178084011,
+      "tokens_seen": 682885120
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038166729714393796,
+      "loss": 2.8264,
+      "theoretical_loss": 3.788232388906935,
+      "tokens_seen": 684195840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038143086816720256,
+      "loss": 2.8892,
+      "theoretical_loss": 3.7874934140009096,
+      "tokens_seen": 685506560
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003811944391904672,
+      "loss": 2.8702,
+      "theoretical_loss": 3.786756245466346,
+      "tokens_seen": 686817280
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038095801021373175,
+      "loss": 2.8296,
+      "theoretical_loss": 3.7860208754530422,
+      "tokens_seen": 688128000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003807215812369964,
+      "loss": 2.8989,
+      "theoretical_loss": 3.785287296159782,
+      "tokens_seen": 689438720
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038048515226026105,
+      "loss": 2.8029,
+      "theoretical_loss": 3.784555499833936,
+      "tokens_seen": 690749440
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003802487232835256,
+      "loss": 2.8094,
+      "theoretical_loss": 3.7838254787710683,
+      "tokens_seen": 692060160
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038001229430679025,
+      "loss": 2.8296,
+      "theoretical_loss": 3.7830972253145463,
+      "tokens_seen": 693370880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037977586533005485,
+      "loss": 2.8966,
+      "theoretical_loss": 3.7823707318551536,
+      "tokens_seen": 694681600
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037953943635331944,
+      "loss": 2.7936,
+      "theoretical_loss": 3.7816459908307074,
+      "tokens_seen": 695992320
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003793030073765841,
+      "loss": 2.8499,
+      "theoretical_loss": 3.7809229947256826,
+      "tokens_seen": 697303040
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003790665783998487,
+      "loss": 2.8423,
+      "theoretical_loss": 3.7802017360708327,
+      "tokens_seen": 698613760
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003788301494231133,
+      "loss": 2.7822,
+      "theoretical_loss": 3.779482207442821,
+      "tokens_seen": 699924480
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003785937204463779,
+      "loss": 2.8023,
+      "theoretical_loss": 3.778764401463854,
+      "tokens_seen": 701235200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037835729146964254,
+      "loss": 2.8137,
+      "theoretical_loss": 3.778048310801316,
+      "tokens_seen": 702545920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037812086249290714,
+      "loss": 2.7984,
+      "theoretical_loss": 3.7773339281674074,
+      "tokens_seen": 703856640
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037788443351617173,
+      "loss": 2.817,
+      "theoretical_loss": 3.7766212463187916,
+      "tokens_seen": 705167360
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003776480045394364,
+      "loss": 2.8796,
+      "theoretical_loss": 3.7759102580562383,
+      "tokens_seen": 706478080
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037741157556270093,
+      "loss": 2.7898,
+      "theoretical_loss": 3.7752009562242757,
+      "tokens_seen": 707788800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003771751465859656,
+      "loss": 2.7628,
+      "theoretical_loss": 3.7744933337108444,
+      "tokens_seen": 709099520
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037693871760923023,
+      "loss": 2.8321,
+      "theoretical_loss": 3.773787383446951,
+      "tokens_seen": 710410240
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003767022886324948,
+      "loss": 2.8259,
+      "theoretical_loss": 3.773083098406333,
+      "tokens_seen": 711720960
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037646585965575943,
+      "loss": 2.8023,
+      "theoretical_loss": 3.7723804716051195,
+      "tokens_seen": 713031680
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000376229430679024,
+      "loss": 2.8084,
+      "theoretical_loss": 3.7716794961014983,
+      "tokens_seen": 714342400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003759930017022886,
+      "loss": 2.7766,
+      "theoretical_loss": 3.770980164995387,
+      "tokens_seen": 715653120
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003757565727255533,
+      "loss": 2.7966,
+      "theoretical_loss": 3.7702824714281062,
+      "tokens_seen": 716963840
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037552014374881787,
+      "loss": 2.7776,
+      "theoretical_loss": 3.7695864085820547,
+      "tokens_seen": 718274560
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037528371477208247,
+      "loss": 2.7785,
+      "theoretical_loss": 3.7688919696803898,
+      "tokens_seen": 719585280
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 405203,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7325472831726074,
+      "objective/train/theoretical_loss": 3.76819914798671,
+      "objective/train/tokens_used": 741356000,
+      "theoretical_loss": 3.76819914798671,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037504728579534707,
+      "loss": 2.7356,
+      "theoretical_loss": 3.76819914798671,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003748108568186117,
+      "loss": 2.8209,
+      "theoretical_loss": 3.7675079368047415,
+      "tokens_seen": 722206720
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037457442784187626,
+      "loss": 2.7781,
+      "theoretical_loss": 3.7668183294780246,
+      "tokens_seen": 723517440
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003743379988651409,
+      "loss": 2.7413,
+      "theoretical_loss": 3.7661303193896076,
+      "tokens_seen": 724828160
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037410156988840556,
+      "loss": 2.7527,
+      "theoretical_loss": 3.76544389996174,
+      "tokens_seen": 726138880
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003738651409116701,
+      "loss": 2.7117,
+      "theoretical_loss": 3.7647590646555713,
+      "tokens_seen": 727449600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037362871193493476,
+      "loss": 2.714,
+      "theoretical_loss": 3.7640758069708493,
+      "tokens_seen": 728760320
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037339228295819936,
+      "loss": 2.7695,
+      "theoretical_loss": 3.763394120445626,
+      "tokens_seen": 730071040
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037315585398146395,
+      "loss": 2.784,
+      "theoretical_loss": 3.7627139986559595,
+      "tokens_seen": 731381760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003729194250047286,
+      "loss": 2.754,
+      "theoretical_loss": 3.7620354352156293,
+      "tokens_seen": 732692480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003726829960279932,
+      "loss": 2.7113,
+      "theoretical_loss": 3.7613584237758415,
+      "tokens_seen": 734003200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003724465670512578,
+      "loss": 2.8187,
+      "theoretical_loss": 3.7606829580249457,
+      "tokens_seen": 735313920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003722101380745224,
+      "loss": 2.8081,
+      "theoretical_loss": 3.760009031688152,
+      "tokens_seen": 736624640
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037197370909778705,
+      "loss": 2.814,
+      "theoretical_loss": 3.7593366385272513,
+      "tokens_seen": 737935360
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037173728012105165,
+      "loss": 2.7847,
+      "theoretical_loss": 3.758665772340336,
+      "tokens_seen": 739246080
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037150085114431625,
+      "loss": 2.8126,
+      "theoretical_loss": 3.7579964269615247,
+      "tokens_seen": 740556800
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003712644221675809,
+      "loss": 2.8318,
+      "theoretical_loss": 3.7573285962606917,
+      "tokens_seen": 741867520
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037102799319084544,
+      "loss": 2.7802,
+      "theoretical_loss": 3.7566622741431956,
+      "tokens_seen": 743178240
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003707915642141101,
+      "loss": 2.8111,
+      "theoretical_loss": 3.75599745454961,
+      "tokens_seen": 744488960
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037055513523737474,
+      "loss": 2.8358,
+      "theoretical_loss": 3.7553341314554625,
+      "tokens_seen": 745799680
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003703187062606393,
+      "loss": 2.8251,
+      "theoretical_loss": 3.754672298870968,
+      "tokens_seen": 747110400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037008227728390394,
+      "loss": 2.8304,
+      "theoretical_loss": 3.754011950840771,
+      "tokens_seen": 748421120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036984584830716854,
+      "loss": 2.8102,
+      "theoretical_loss": 3.753353081443688,
+      "tokens_seen": 749731840
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036960941933043313,
+      "loss": 2.7869,
+      "theoretical_loss": 3.752695684792449,
+      "tokens_seen": 751042560
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036937299035369773,
+      "loss": 2.7899,
+      "theoretical_loss": 3.752039755033449,
+      "tokens_seen": 752353280
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003691365613769624,
+      "loss": 2.7801,
+      "theoretical_loss": 3.751385286346493,
+      "tokens_seen": 753664000
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000368900132400227,
+      "loss": 2.7465,
+      "theoretical_loss": 3.7507322729445525,
+      "tokens_seen": 754974720
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003686637034234916,
+      "loss": 2.7581,
+      "theoretical_loss": 3.7500807090735115,
+      "tokens_seen": 756285440
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036842727444675623,
+      "loss": 2.7208,
+      "theoretical_loss": 3.749430589011933,
+      "tokens_seen": 757596160
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036819084547002077,
+      "loss": 2.7744,
+      "theoretical_loss": 3.748781907070807,
+      "tokens_seen": 758906880
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679544164932854,
+      "loss": 2.8423,
+      "theoretical_loss": 3.7481346575933197,
+      "tokens_seen": 760217600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003677179875165501,
+      "loss": 2.8088,
+      "theoretical_loss": 3.747488834954609,
+      "tokens_seen": 761528320
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003674815585398146,
+      "loss": 2.8077,
+      "theoretical_loss": 3.7468444335615363,
+      "tokens_seen": 762839040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036724512956307927,
+      "loss": 2.7254,
+      "theoretical_loss": 3.746201447852446,
+      "tokens_seen": 764149760
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036700870058634387,
+      "loss": 2.7605,
+      "theoretical_loss": 3.745559872296941,
+      "tokens_seen": 765460480
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036677227160960847,
+      "loss": 2.7344,
+      "theoretical_loss": 3.7449197013956503,
+      "tokens_seen": 766771200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003665358426328731,
+      "loss": 2.6958,
+      "theoretical_loss": 3.744280929680005,
+      "tokens_seen": 768081920
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003662994136561377,
+      "loss": 2.7527,
+      "theoretical_loss": 3.743643551712009,
+      "tokens_seen": 769392640
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003660629846794023,
+      "loss": 2.7773,
+      "theoretical_loss": 3.7430075620840215,
+      "tokens_seen": 770703360
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003658265557026669,
+      "loss": 2.7209,
+      "theoretical_loss": 3.7423729554185337,
+      "tokens_seen": 772014080
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036559012672593156,
+      "loss": 2.7133,
+      "theoretical_loss": 3.7417397263679493,
+      "tokens_seen": 773324800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036535369774919616,
+      "loss": 2.799,
+      "theoretical_loss": 3.7411078696143694,
+      "tokens_seen": 774635520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036511726877246076,
+      "loss": 2.8186,
+      "theoretical_loss": 3.740477379869378,
+      "tokens_seen": 775946240
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003648808397957254,
+      "loss": 2.7996,
+      "theoretical_loss": 3.739848251873827,
+      "tokens_seen": 777256960
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036464441081898995,
+      "loss": 2.8178,
+      "theoretical_loss": 3.739220480397626,
+      "tokens_seen": 778567680
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003644079818422546,
+      "loss": 2.7482,
+      "theoretical_loss": 3.7385940602395347,
+      "tokens_seen": 779878400
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036417155286551926,
+      "loss": 2.7766,
+      "theoretical_loss": 3.737968986226954,
+      "tokens_seen": 781189120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003639351238887838,
+      "loss": 2.7678,
+      "theoretical_loss": 3.7373452532157208,
+      "tokens_seen": 782499840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036369869491204845,
+      "loss": 2.8746,
+      "theoretical_loss": 3.736722856089904,
+      "tokens_seen": 783810560
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036346226593531305,
+      "loss": 2.8423,
+      "theoretical_loss": 3.736101789761605,
+      "tokens_seen": 785121280
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 440553,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.451080322265625,
+      "objective/train/theoretical_loss": 3.735482049170753,
+      "objective/train/tokens_used": 806892000,
+      "theoretical_loss": 3.735482049170753,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036322583695857765,
+      "loss": 2.7965,
+      "theoretical_loss": 3.735482049170753,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036298940798184224,
+      "loss": 2.8513,
+      "theoretical_loss": 3.734863629284911,
+      "tokens_seen": 787742720
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003627529790051069,
+      "loss": 2.8257,
+      "theoretical_loss": 3.7342465250990777,
+      "tokens_seen": 789053440
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003625165500283715,
+      "loss": 2.8062,
+      "theoretical_loss": 3.7336307316354933,
+      "tokens_seen": 790364160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003622801210516361,
+      "loss": 2.8369,
+      "theoretical_loss": 3.7330162439434442,
+      "tokens_seen": 791674880
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036204369207490074,
+      "loss": 2.8066,
+      "theoretical_loss": 3.7324030570990736,
+      "tokens_seen": 792985600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003618072630981653,
+      "loss": 2.8327,
+      "theoretical_loss": 3.7317911662051912,
+      "tokens_seen": 794296320
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036157083412142994,
+      "loss": 2.7777,
+      "theoretical_loss": 3.731180566391086,
+      "tokens_seen": 795607040
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003613344051446946,
+      "loss": 2.7566,
+      "theoretical_loss": 3.7305712528123367,
+      "tokens_seen": 796917760
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036109797616795913,
+      "loss": 2.8123,
+      "theoretical_loss": 3.729963220650631,
+      "tokens_seen": 798228480
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003608615471912238,
+      "loss": 2.8019,
+      "theoretical_loss": 3.729356465113578,
+      "tokens_seen": 799539200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003606251182144884,
+      "loss": 2.8193,
+      "theoretical_loss": 3.7287509814345317,
+      "tokens_seen": 800849920
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000360388689237753,
+      "loss": 2.8706,
+      "theoretical_loss": 3.7281467648724043,
+      "tokens_seen": 802160640
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036015226026101763,
+      "loss": 2.7808,
+      "theoretical_loss": 3.727543810711495,
+      "tokens_seen": 803471360
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003599158312842822,
+      "loss": 2.8134,
+      "theoretical_loss": 3.7269421142613055,
+      "tokens_seen": 804782080
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003596794023075468,
+      "loss": 2.8048,
+      "theoretical_loss": 3.7263416708563715,
+      "tokens_seen": 806092800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003594429733308114,
+      "loss": 2.7764,
+      "theoretical_loss": 3.725742475856083,
+      "tokens_seen": 807403520
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003592065443540761,
+      "loss": 2.8125,
+      "theoretical_loss": 3.7251445246445147,
+      "tokens_seen": 808714240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035897011537734067,
+      "loss": 2.8072,
+      "theoretical_loss": 3.724547812630255,
+      "tokens_seen": 810024960
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035873368640060527,
+      "loss": 2.8413,
+      "theoretical_loss": 3.7239523352462354,
+      "tokens_seen": 811335680
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003584972574238699,
+      "loss": 2.8219,
+      "theoretical_loss": 3.7233580879495616,
+      "tokens_seen": 812646400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035826082844713446,
+      "loss": 2.7908,
+      "theoretical_loss": 3.72276506622135,
+      "tokens_seen": 813957120
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003580243994703991,
+      "loss": 2.7585,
+      "theoretical_loss": 3.722173265566559,
+      "tokens_seen": 815267840
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035778797049366366,
+      "loss": 2.8308,
+      "theoretical_loss": 3.7215826815138264,
+      "tokens_seen": 816578560
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003575515415169283,
+      "loss": 2.8019,
+      "theoretical_loss": 3.720993309615307,
+      "tokens_seen": 817889280
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035731511254019296,
+      "loss": 2.8033,
+      "theoretical_loss": 3.720405145446512,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003570786835634575,
+      "loss": 2.793,
+      "theoretical_loss": 3.719818184606148,
+      "tokens_seen": 820510720
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035684225458672216,
+      "loss": 2.8288,
+      "theoretical_loss": 3.719232422715959,
+      "tokens_seen": 821821440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035660582560998675,
+      "loss": 2.7718,
+      "theoretical_loss": 3.7186478554205693,
+      "tokens_seen": 823132160
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003563812180820881,
+      "loss": 2.7668,
+      "theoretical_loss": 3.718064478387329,
+      "tokens_seen": 824442880
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035614478910535275,
+      "loss": 2.7649,
+      "theoretical_loss": 3.717482287306157,
+      "tokens_seen": 825753600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003559083601286174,
+      "loss": 2.7729,
+      "theoretical_loss": 3.7169012778893897,
+      "tokens_seen": 827064320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035567193115188195,
+      "loss": 2.7556,
+      "theoretical_loss": 3.71632144587163,
+      "tokens_seen": 828375040
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003554355021751466,
+      "loss": 2.8103,
+      "theoretical_loss": 3.7157427870095936,
+      "tokens_seen": 829685760
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003551990731984112,
+      "loss": 2.8171,
+      "theoretical_loss": 3.7151652970819615,
+      "tokens_seen": 830996480
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003549626442216758,
+      "loss": 2.761,
+      "theoretical_loss": 3.714588971889232,
+      "tokens_seen": 832307200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003547262152449404,
+      "loss": 2.7281,
+      "theoretical_loss": 3.7140138072535738,
+      "tokens_seen": 833617920
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035448978626820504,
+      "loss": 2.7305,
+      "theoretical_loss": 3.7134397990186794,
+      "tokens_seen": 834928640
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035425335729146964,
+      "loss": 2.7776,
+      "theoretical_loss": 3.7128669430496215,
+      "tokens_seen": 836239360
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035401692831473424,
+      "loss": 2.7347,
+      "theoretical_loss": 3.71229523523271,
+      "tokens_seen": 837550080
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003537804993379989,
+      "loss": 2.6805,
+      "theoretical_loss": 3.7117246714753485,
+      "tokens_seen": 838860800
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035354407036126343,
+      "loss": 2.7383,
+      "theoretical_loss": 3.7111552477058947,
+      "tokens_seen": 840171520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003533076413845281,
+      "loss": 2.7158,
+      "theoretical_loss": 3.7105869598735195,
+      "tokens_seen": 841482240
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035307121240779274,
+      "loss": 2.7225,
+      "theoretical_loss": 3.71001980394807,
+      "tokens_seen": 842792960
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003528347834310573,
+      "loss": 2.7274,
+      "theoretical_loss": 3.709453775919931,
+      "tokens_seen": 844103680
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035259835445432193,
+      "loss": 2.7399,
+      "theoretical_loss": 3.7088888717998865,
+      "tokens_seen": 845414400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035236192547758653,
+      "loss": 2.7129,
+      "theoretical_loss": 3.70832508761899,
+      "tokens_seen": 846725120
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035212549650085113,
+      "loss": 2.7134,
+      "theoretical_loss": 3.707762419428425,
+      "tokens_seen": 848035840
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003518890675241158,
+      "loss": 2.7313,
+      "theoretical_loss": 3.7072008632993745,
+      "tokens_seen": 849346560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003516526385473804,
+      "loss": 2.8078,
+      "theoretical_loss": 3.7066404153228873,
+      "tokens_seen": 850657280
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 477195,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.859386920928955,
+      "objective/train/theoretical_loss": 3.7060810716097503,
+      "objective/train/tokens_used": 872428000,
+      "theoretical_loss": 3.7060810716097503,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000351416209570645,
+      "loss": 2.7659,
+      "theoretical_loss": 3.7060810716097503,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035117978059390957,
+      "loss": 2.7479,
+      "theoretical_loss": 3.7055228282903556,
+      "tokens_seen": 853278720
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003509433516171742,
+      "loss": 2.7731,
+      "theoretical_loss": 3.704965681514573,
+      "tokens_seen": 854589440
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003507069226404388,
+      "loss": 2.8253,
+      "theoretical_loss": 3.7044096274516214,
+      "tokens_seen": 855900160
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003504704936637034,
+      "loss": 2.716,
+      "theoretical_loss": 3.7038546622899435,
+      "tokens_seen": 857210880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035023406468696807,
+      "loss": 2.7689,
+      "theoretical_loss": 3.70330078223708,
+      "tokens_seen": 858521600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003499976357102326,
+      "loss": 2.7538,
+      "theoretical_loss": 3.7027479835195427,
+      "tokens_seen": 859832320
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034976120673349727,
+      "loss": 2.7398,
+      "theoretical_loss": 3.702196262382694,
+      "tokens_seen": 861143040
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003495247777567619,
+      "loss": 2.7677,
+      "theoretical_loss": 3.701645615090622,
+      "tokens_seen": 862453760
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034928834878002646,
+      "loss": 2.6571,
+      "theoretical_loss": 3.7010960379260185,
+      "tokens_seen": 863764480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003490519198032911,
+      "loss": 2.7121,
+      "theoretical_loss": 3.7005475271900616,
+      "tokens_seen": 865075200
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003488154908265557,
+      "loss": 2.7246,
+      "theoretical_loss": 3.7000000792022916,
+      "tokens_seen": 866385920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003485790618498203,
+      "loss": 2.6893,
+      "theoretical_loss": 3.699453690300495,
+      "tokens_seen": 867696640
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003483426328730849,
+      "loss": 2.7052,
+      "theoretical_loss": 3.698908356840586,
+      "tokens_seen": 869007360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034810620389634956,
+      "loss": 2.7138,
+      "theoretical_loss": 3.6983640751964897,
+      "tokens_seen": 870318080
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034786977491961415,
+      "loss": 2.774,
+      "theoretical_loss": 3.697820841760025,
+      "tokens_seen": 871628800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034763334594287875,
+      "loss": 2.738,
+      "theoretical_loss": 3.697278652940791,
+      "tokens_seen": 872939520
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003473969169661434,
+      "loss": 2.74,
+      "theoretical_loss": 3.6967375051660527,
+      "tokens_seen": 874250240
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034716048798940795,
+      "loss": 2.7524,
+      "theoretical_loss": 3.6961973948806266,
+      "tokens_seen": 875560960
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003469240590126726,
+      "loss": 2.7544,
+      "theoretical_loss": 3.6956583185467693,
+      "tokens_seen": 876871680
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034668763003593725,
+      "loss": 2.6895,
+      "theoretical_loss": 3.6951202726440666,
+      "tokens_seen": 878182400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003464512010592018,
+      "loss": 2.7112,
+      "theoretical_loss": 3.6945832536693204,
+      "tokens_seen": 879493120
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034621477208246644,
+      "loss": 2.7563,
+      "theoretical_loss": 3.6940472581364423,
+      "tokens_seen": 880803840
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034597834310573104,
+      "loss": 2.6831,
+      "theoretical_loss": 3.6935122825763433,
+      "tokens_seen": 882114560
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034574191412899564,
+      "loss": 2.7634,
+      "theoretical_loss": 3.692978323536824,
+      "tokens_seen": 883425280
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003455054851522603,
+      "loss": 2.7442,
+      "theoretical_loss": 3.6924453775824704,
+      "tokens_seen": 884736000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003452690561755249,
+      "loss": 2.7676,
+      "theoretical_loss": 3.6919134412945462,
+      "tokens_seen": 886046720
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003450326271987895,
+      "loss": 2.798,
+      "theoretical_loss": 3.691382511270887,
+      "tokens_seen": 887357440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003447961982220541,
+      "loss": 2.7513,
+      "theoretical_loss": 3.6908525841257953,
+      "tokens_seen": 888668160
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034455976924531874,
+      "loss": 2.7525,
+      "theoretical_loss": 3.6903236564899395,
+      "tokens_seen": 889978880
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034432334026858333,
+      "loss": 2.7335,
+      "theoretical_loss": 3.6897957250102467,
+      "tokens_seen": 891289600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034408691129184793,
+      "loss": 2.6977,
+      "theoretical_loss": 3.6892687863498024,
+      "tokens_seen": 892600320
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003438504823151126,
+      "loss": 2.7333,
+      "theoretical_loss": 3.6887428371877506,
+      "tokens_seen": 893911040
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003436140533383771,
+      "loss": 2.7406,
+      "theoretical_loss": 3.688217874219191,
+      "tokens_seen": 895221760
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003433776243616418,
+      "loss": 2.7368,
+      "theoretical_loss": 3.6876938941550783,
+      "tokens_seen": 896532480
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034314119538490643,
+      "loss": 2.7568,
+      "theoretical_loss": 3.6871708937221275,
+      "tokens_seen": 897843200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034290476640817097,
+      "loss": 2.7438,
+      "theoretical_loss": 3.686648869662709,
+      "tokens_seen": 899153920
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003426683374314356,
+      "loss": 2.7176,
+      "theoretical_loss": 3.6861278187347573,
+      "tokens_seen": 900464640
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003424319084547002,
+      "loss": 2.7278,
+      "theoretical_loss": 3.6856077377116696,
+      "tokens_seen": 901775360
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003421954794779648,
+      "loss": 2.704,
+      "theoretical_loss": 3.685088623382213,
+      "tokens_seen": 903086080
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003419590505012294,
+      "loss": 2.6932,
+      "theoretical_loss": 3.6845704725504254,
+      "tokens_seen": 904396800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034172262152449407,
+      "loss": 2.6636,
+      "theoretical_loss": 3.684053282035526,
+      "tokens_seen": 905707520
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034148619254775866,
+      "loss": 2.7068,
+      "theoretical_loss": 3.6835370486718153,
+      "tokens_seen": 907018240
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034124976357102326,
+      "loss": 2.7572,
+      "theoretical_loss": 3.6830217693085876,
+      "tokens_seen": 908328960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003410133345942879,
+      "loss": 2.7177,
+      "theoretical_loss": 3.682507440810034,
+      "tokens_seen": 909639680
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034077690561755246,
+      "loss": 2.6885,
+      "theoretical_loss": 3.6819940600551533,
+      "tokens_seen": 910950400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003405404766408171,
+      "loss": 2.7888,
+      "theoretical_loss": 3.6814816239376595,
+      "tokens_seen": 912261120
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034030404766408176,
+      "loss": 2.7462,
+      "theoretical_loss": 3.6809701293658934,
+      "tokens_seen": 913571840
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003400676186873463,
+      "loss": 2.7734,
+      "theoretical_loss": 3.6804595732627297,
+      "tokens_seen": 914882560
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033983118971061096,
+      "loss": 2.7436,
+      "theoretical_loss": 3.6799499525654915,
+      "tokens_seen": 916193280
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 513163,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.83975887298584,
+      "objective/train/theoretical_loss": 3.6794412642258583,
+      "objective/train/tokens_used": 937964000,
+      "theoretical_loss": 3.6794412642258583,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033959476073387555,
+      "loss": 2.7804,
+      "theoretical_loss": 3.6794412642258583,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033935833175714015,
+      "loss": 2.787,
+      "theoretical_loss": 3.6789335052097814,
+      "tokens_seen": 918814720
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003391219027804048,
+      "loss": 2.7717,
+      "theoretical_loss": 3.6784266724973946,
+      "tokens_seen": 920125440
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003388854738036694,
+      "loss": 2.7296,
+      "theoretical_loss": 3.6779207630829296,
+      "tokens_seen": 921436160
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000338649044826934,
+      "loss": 2.7581,
+      "theoretical_loss": 3.677415773974629,
+      "tokens_seen": 922746880
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003384126158501986,
+      "loss": 2.726,
+      "theoretical_loss": 3.6769117021946625,
+      "tokens_seen": 924057600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033817618687346325,
+      "loss": 2.7613,
+      "theoretical_loss": 3.676408544779039,
+      "tokens_seen": 925368320
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033793975789672784,
+      "loss": 2.7116,
+      "theoretical_loss": 3.6759062987775293,
+      "tokens_seen": 926679040
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033770332891999244,
+      "loss": 2.7743,
+      "theoretical_loss": 3.6754049612535753,
+      "tokens_seen": 927989760
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003374668999432571,
+      "loss": 2.7662,
+      "theoretical_loss": 3.6749045292842126,
+      "tokens_seen": 929300480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033723047096652164,
+      "loss": 2.7676,
+      "theoretical_loss": 3.674404999959987,
+      "tokens_seen": 930611200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003369940419897863,
+      "loss": 2.8034,
+      "theoretical_loss": 3.6739063703848718,
+      "tokens_seen": 931921920
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003367576130130509,
+      "loss": 2.7715,
+      "theoretical_loss": 3.6734086376761907,
+      "tokens_seen": 933232640
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003365211840363155,
+      "loss": 2.8154,
+      "theoretical_loss": 3.6729117989645332,
+      "tokens_seen": 934543360
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033628475505958013,
+      "loss": 2.7405,
+      "theoretical_loss": 3.6724158513936773,
+      "tokens_seen": 935854080
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033604832608284473,
+      "loss": 2.7108,
+      "theoretical_loss": 3.6719207921205115,
+      "tokens_seen": 937164800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033581189710610933,
+      "loss": 2.7059,
+      "theoretical_loss": 3.6714266183149533,
+      "tokens_seen": 938475520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003355754681293739,
+      "loss": 2.7555,
+      "theoretical_loss": 3.670933327159875,
+      "tokens_seen": 939786240
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003353390391526386,
+      "loss": 2.7353,
+      "theoretical_loss": 3.670440915851024,
+      "tokens_seen": 941096960
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003351026101759032,
+      "loss": 2.73,
+      "theoretical_loss": 3.6699493815969464,
+      "tokens_seen": 942407680
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003348661811991678,
+      "loss": 2.7755,
+      "theoretical_loss": 3.6694587216189114,
+      "tokens_seen": 943718400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003346297522224324,
+      "loss": 2.7286,
+      "theoretical_loss": 3.6689689331508366,
+      "tokens_seen": 945029120
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033439332324569697,
+      "loss": 2.7363,
+      "theoretical_loss": 3.6684800134392104,
+      "tokens_seen": 946339840
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003341568942689616,
+      "loss": 2.7026,
+      "theoretical_loss": 3.6679919597430213,
+      "tokens_seen": 947650560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033392046529222627,
+      "loss": 2.6754,
+      "theoretical_loss": 3.6675047693336795,
+      "tokens_seen": 948961280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003336840363154908,
+      "loss": 2.7145,
+      "theoretical_loss": 3.6670184394949477,
+      "tokens_seen": 950272000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033344760733875547,
+      "loss": 2.7392,
+      "theoretical_loss": 3.6665329675228646,
+      "tokens_seen": 951582720
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033321117836202,
+      "loss": 2.7079,
+      "theoretical_loss": 3.666048350725677,
+      "tokens_seen": 952893440
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033297474938528466,
+      "loss": 2.7194,
+      "theoretical_loss": 3.6655645864237627,
+      "tokens_seen": 954204160
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003327383204085493,
+      "loss": 2.6784,
+      "theoretical_loss": 3.6650816719495625,
+      "tokens_seen": 955514880
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033250189143181386,
+      "loss": 2.6533,
+      "theoretical_loss": 3.6645996046475107,
+      "tokens_seen": 956825600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003322654624550785,
+      "loss": 2.6745,
+      "theoretical_loss": 3.664118381873961,
+      "tokens_seen": 958136320
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003320290334783431,
+      "loss": 2.665,
+      "theoretical_loss": 3.66363800099712,
+      "tokens_seen": 959447040
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003317926045016077,
+      "loss": 2.7108,
+      "theoretical_loss": 3.6631584593969757,
+      "tokens_seen": 960757760
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003315561755248723,
+      "loss": 2.648,
+      "theoretical_loss": 3.6626797544652305,
+      "tokens_seen": 962068480
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033131974654813695,
+      "loss": 2.6699,
+      "theoretical_loss": 3.6622018836052317,
+      "tokens_seen": 963379200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033108331757140155,
+      "loss": 2.7234,
+      "theoretical_loss": 3.6617248442319053,
+      "tokens_seen": 964689920
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033084688859466615,
+      "loss": 2.6963,
+      "theoretical_loss": 3.661248633771687,
+      "tokens_seen": 966000640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003306104596179308,
+      "loss": 2.6768,
+      "theoretical_loss": 3.6607732496624568,
+      "tokens_seen": 967311360
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033037403064119534,
+      "loss": 2.6924,
+      "theoretical_loss": 3.660298689353471,
+      "tokens_seen": 968622080
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033013760166446,
+      "loss": 2.7223,
+      "theoretical_loss": 3.659824950305297,
+      "tokens_seen": 969932800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032990117268772465,
+      "loss": 2.7665,
+      "theoretical_loss": 3.6593520299897513,
+      "tokens_seen": 971243520
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003296647437109892,
+      "loss": 2.7692,
+      "theoretical_loss": 3.6588799258898277,
+      "tokens_seen": 972554240
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032942831473425384,
+      "loss": 2.7858,
+      "theoretical_loss": 3.65840863549964,
+      "tokens_seen": 973864960
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032919188575751844,
+      "loss": 2.737,
+      "theoretical_loss": 3.6579381563243527,
+      "tokens_seen": 975175680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032895545678078304,
+      "loss": 2.7019,
+      "theoretical_loss": 3.6574684858801207,
+      "tokens_seen": 976486400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003287190278040477,
+      "loss": 2.6586,
+      "theoretical_loss": 3.656999621694025,
+      "tokens_seen": 977797120
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003284825988273123,
+      "loss": 2.6835,
+      "theoretical_loss": 3.65653156130401,
+      "tokens_seen": 979107840
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003282461698505769,
+      "loss": 2.6699,
+      "theoretical_loss": 3.6560643022588217,
+      "tokens_seen": 980418560
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003280097408738415,
+      "loss": 2.6888,
+      "theoretical_loss": 3.655597842117945,
+      "tokens_seen": 981729280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5395508255365915,
+      "debugging/distinct-1-grams": 0.7683501269789897,
+      "debugging/distinct-2-grams": 0.9413977022884555,
+      "debugging/entropy-1-grams": 6.198614472060548,
+      "debugging/entropy-2-grams": 7.1599602011918995,
+      "debugging/length": 502.2,
+      "debugging/num_segments": 20,
+      "epoch": 0.35,
+      "objective/train/docs_used": 548797,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.6332430839538574,
+      "objective/train/theoretical_loss": 3.6551321784515434,
+      "objective/train/tokens_used": 1003500000,
+      "theoretical_loss": 3.6551321784515434,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032777331189710613,
+      "loss": 2.6951,
+      "theoretical_loss": 3.6551321784515434,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032753688292037073,
+      "loss": 2.7164,
+      "theoretical_loss": 3.6546673088403985,
+      "tokens_seen": 984350720
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003273004539436353,
+      "loss": 2.7035,
+      "theoretical_loss": 3.654203230875848,
+      "tokens_seen": 985661440
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003270640249669,
+      "loss": 2.6849,
+      "theoretical_loss": 3.653739942159728,
+      "tokens_seen": 986972160
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003268275959901645,
+      "loss": 2.6898,
+      "theoretical_loss": 3.65327744030431,
+      "tokens_seen": 988282880
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003265911670134292,
+      "loss": 2.7672,
+      "theoretical_loss": 3.6528157229322447,
+      "tokens_seen": 989593600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003263547380366938,
+      "loss": 2.6891,
+      "theoretical_loss": 3.6523547876765017,
+      "tokens_seen": 990904320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003261419519576319,
+      "loss": 2.7356,
+      "theoretical_loss": 3.651894632180312,
+      "tokens_seen": 992215040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003259055229808965,
+      "loss": 2.8215,
+      "theoretical_loss": 3.65143525409711,
+      "tokens_seen": 993525760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003256690940041612,
+      "loss": 2.7848,
+      "theoretical_loss": 3.650976651090474,
+      "tokens_seen": 994836480
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032543266502742577,
+      "loss": 2.8296,
+      "theoretical_loss": 3.650518820834072,
+      "tokens_seen": 996147200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032519623605069037,
+      "loss": 2.8453,
+      "theoretical_loss": 3.650061761011603,
+      "tokens_seen": 997457920
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000324959807073955,
+      "loss": 2.8247,
+      "theoretical_loss": 3.649605469316742,
+      "tokens_seen": 998768640
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032472337809721956,
+      "loss": 2.8156,
+      "theoretical_loss": 3.6491499434530814,
+      "tokens_seen": 1000079360
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003244869491204842,
+      "loss": 2.8627,
+      "theoretical_loss": 3.648695181134078,
+      "tokens_seen": 1001390080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003242505201437488,
+      "loss": 2.7878,
+      "theoretical_loss": 3.648241180082998,
+      "tokens_seen": 1002700800
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003240140911670134,
+      "loss": 2.8126,
+      "theoretical_loss": 3.64778793803286,
+      "tokens_seen": 1004011520
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032377766219027806,
+      "loss": 2.8221,
+      "theoretical_loss": 3.6473354527263813,
+      "tokens_seen": 1005322240
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032354123321354266,
+      "loss": 2.8165,
+      "theoretical_loss": 3.6468837219159256,
+      "tokens_seen": 1006632960
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032330480423680726,
+      "loss": 2.844,
+      "theoretical_loss": 3.6464327433634467,
+      "tokens_seen": 1007943680
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032306837526007185,
+      "loss": 2.8321,
+      "theoretical_loss": 3.645982514840437,
+      "tokens_seen": 1009254400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003228319462833365,
+      "loss": 2.7851,
+      "theoretical_loss": 3.6455330341278733,
+      "tokens_seen": 1010565120
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003225955173066011,
+      "loss": 2.8547,
+      "theoretical_loss": 3.645084299016166,
+      "tokens_seen": 1011875840
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003223590883298657,
+      "loss": 2.8491,
+      "theoretical_loss": 3.644636307305104,
+      "tokens_seen": 1013186560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032212265935313035,
+      "loss": 2.778,
+      "theoretical_loss": 3.644189056803805,
+      "tokens_seen": 1014497280
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003218862303763949,
+      "loss": 2.8114,
+      "theoretical_loss": 3.6437425453306638,
+      "tokens_seen": 1015808000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032164980139965955,
+      "loss": 2.8036,
+      "theoretical_loss": 3.643296770713301,
+      "tokens_seen": 1017118720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003214133724229242,
+      "loss": 2.7573,
+      "theoretical_loss": 3.642851730788511,
+      "tokens_seen": 1018429440
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032117694344618874,
+      "loss": 2.7723,
+      "theoretical_loss": 3.642407423402213,
+      "tokens_seen": 1019740160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003209405144694534,
+      "loss": 2.7911,
+      "theoretical_loss": 3.641963846409399,
+      "tokens_seen": 1021050880
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000320704085492718,
+      "loss": 2.7345,
+      "theoretical_loss": 3.641520997674087,
+      "tokens_seen": 1022361600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003204676565159826,
+      "loss": 2.796,
+      "theoretical_loss": 3.641078875069269,
+      "tokens_seen": 1023672320
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032023122753924724,
+      "loss": 2.7403,
+      "theoretical_loss": 3.640637476476862,
+      "tokens_seen": 1024983040
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031999479856251184,
+      "loss": 2.7171,
+      "theoretical_loss": 3.64019679978766,
+      "tokens_seen": 1026293760
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031975836958577644,
+      "loss": 2.7919,
+      "theoretical_loss": 3.639756842901286,
+      "tokens_seen": 1027604480
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031952194060904103,
+      "loss": 2.7861,
+      "theoretical_loss": 3.639317603726142,
+      "tokens_seen": 1028915200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003192855116323057,
+      "loss": 2.858,
+      "theoretical_loss": 3.638879080179364,
+      "tokens_seen": 1030225920
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031904908265557023,
+      "loss": 2.7529,
+      "theoretical_loss": 3.63844127018677,
+      "tokens_seen": 1031536640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003188126536788349,
+      "loss": 2.7633,
+      "theoretical_loss": 3.6380041716828186,
+      "tokens_seen": 1032847360
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031857622470209953,
+      "loss": 2.8117,
+      "theoretical_loss": 3.637567782610558,
+      "tokens_seen": 1034158080
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003183397957253641,
+      "loss": 2.7669,
+      "theoretical_loss": 3.637132100921579,
+      "tokens_seen": 1035468800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003181033667486287,
+      "loss": 2.752,
+      "theoretical_loss": 3.6366971245759734,
+      "tokens_seen": 1036779520
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003178669377718933,
+      "loss": 2.7997,
+      "theoretical_loss": 3.6362628515422815,
+      "tokens_seen": 1038090240
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003176305087951579,
+      "loss": 2.8115,
+      "theoretical_loss": 3.635829279797453,
+      "tokens_seen": 1039400960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003173940798184226,
+      "loss": 2.811,
+      "theoretical_loss": 3.6353964073267964,
+      "tokens_seen": 1040711680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031715765084168717,
+      "loss": 2.814,
+      "theoretical_loss": 3.6349642321239375,
+      "tokens_seen": 1042022400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031692122186495177,
+      "loss": 2.7682,
+      "theoretical_loss": 3.6345327521907724,
+      "tokens_seen": 1043333120
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031668479288821637,
+      "loss": 2.7907,
+      "theoretical_loss": 3.6341019655374245,
+      "tokens_seen": 1044643840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000316448363911481,
+      "loss": 2.7818,
+      "theoretical_loss": 3.6336718701821997,
+      "tokens_seen": 1045954560
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003162119349347456,
+      "loss": 2.7907,
+      "theoretical_loss": 3.633242464151543,
+      "tokens_seen": 1047265280
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 582102,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.738445281982422,
+      "objective/train/theoretical_loss": 3.632813745479994,
+      "objective/train/tokens_used": 1069036000,
+      "theoretical_loss": 3.632813745479994,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003159755059580102,
+      "loss": 2.8192,
+      "theoretical_loss": 3.632813745479994,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031573907698127486,
+      "loss": 2.7641,
+      "theoretical_loss": 3.6323857122101444,
+      "tokens_seen": 1049886720
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003155026480045394,
+      "loss": 2.7563,
+      "theoretical_loss": 3.631958362392595,
+      "tokens_seen": 1051197440
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031526621902780406,
+      "loss": 2.809,
+      "theoretical_loss": 3.631531694085914,
+      "tokens_seen": 1052508160
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003150297900510687,
+      "loss": 2.7815,
+      "theoretical_loss": 3.6311057053565907,
+      "tokens_seen": 1053818880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031479336107433325,
+      "loss": 2.7622,
+      "theoretical_loss": 3.6306803942789987,
+      "tokens_seen": 1055129600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003145569320975979,
+      "loss": 2.8323,
+      "theoretical_loss": 3.63025575893535,
+      "tokens_seen": 1056440320
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003143205031208625,
+      "loss": 2.7748,
+      "theoretical_loss": 3.6298317974156555,
+      "tokens_seen": 1057751040
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003140840741441271,
+      "loss": 2.7607,
+      "theoretical_loss": 3.629408507817682,
+      "tokens_seen": 1059061760
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031384764516739175,
+      "loss": 2.7514,
+      "theoretical_loss": 3.6289858882469135,
+      "tokens_seen": 1060372480
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031361121619065635,
+      "loss": 2.7373,
+      "theoretical_loss": 3.6285639368165077,
+      "tokens_seen": 1061683200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031337478721392095,
+      "loss": 2.7827,
+      "theoretical_loss": 3.6281426516472584,
+      "tokens_seen": 1062993920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031313835823718554,
+      "loss": 2.7113,
+      "theoretical_loss": 3.627722030867552,
+      "tokens_seen": 1064304640
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003129019292604502,
+      "loss": 2.7309,
+      "theoretical_loss": 3.62730207261333,
+      "tokens_seen": 1065615360
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031266550028371474,
+      "loss": 2.708,
+      "theoretical_loss": 3.626882775028048,
+      "tokens_seen": 1066926080
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003124290713069794,
+      "loss": 2.7269,
+      "theoretical_loss": 3.6264641362626375,
+      "tokens_seen": 1068236800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031219264233024404,
+      "loss": 2.7027,
+      "theoretical_loss": 3.626046154475465,
+      "tokens_seen": 1069547520
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003119562133535086,
+      "loss": 2.7199,
+      "theoretical_loss": 3.625628827832294,
+      "tokens_seen": 1070858240
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031171978437677324,
+      "loss": 2.7021,
+      "theoretical_loss": 3.625212154506246,
+      "tokens_seen": 1072168960
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031148335540003784,
+      "loss": 2.7076,
+      "theoretical_loss": 3.624796132677763,
+      "tokens_seen": 1073479680
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031124692642330243,
+      "loss": 2.6884,
+      "theoretical_loss": 3.624380760534567,
+      "tokens_seen": 1074790400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003110104974465671,
+      "loss": 2.7631,
+      "theoretical_loss": 3.6239660362716246,
+      "tokens_seen": 1076101120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003107740684698317,
+      "loss": 2.7291,
+      "theoretical_loss": 3.623551958091108,
+      "tokens_seen": 1077411840
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003105376394930963,
+      "loss": 2.7378,
+      "theoretical_loss": 3.623138524202358,
+      "tokens_seen": 1078722560
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003103012105163609,
+      "loss": 2.7292,
+      "theoretical_loss": 3.622725732821844,
+      "tokens_seen": 1080033280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00031006478153962553,
+      "loss": 2.7589,
+      "theoretical_loss": 3.622313582173133,
+      "tokens_seen": 1081344000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003098283525628901,
+      "loss": 2.6946,
+      "theoretical_loss": 3.621902070486846,
+      "tokens_seen": 1082654720
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003095919235861547,
+      "loss": 2.7934,
+      "theoretical_loss": 3.6214911960006275,
+      "tokens_seen": 1083965440
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003093554946094194,
+      "loss": 2.7683,
+      "theoretical_loss": 3.621080956959104,
+      "tokens_seen": 1085276160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003091190656326839,
+      "loss": 2.6777,
+      "theoretical_loss": 3.6206713516138507,
+      "tokens_seen": 1086586880
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030888263665594857,
+      "loss": 2.7258,
+      "theoretical_loss": 3.620262378223357,
+      "tokens_seen": 1087897600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003086462076792132,
+      "loss": 2.7544,
+      "theoretical_loss": 3.619854035052987,
+      "tokens_seen": 1089208320
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030840977870247777,
+      "loss": 2.7211,
+      "theoretical_loss": 3.619446320374947,
+      "tokens_seen": 1090519040
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003081733497257424,
+      "loss": 2.7215,
+      "theoretical_loss": 3.61903923246825,
+      "tokens_seen": 1091829760
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000307936920749007,
+      "loss": 2.6715,
+      "theoretical_loss": 3.6186327696186797,
+      "tokens_seen": 1093140480
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003077004917722716,
+      "loss": 2.7124,
+      "theoretical_loss": 3.618226930118757,
+      "tokens_seen": 1094451200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030746406279553626,
+      "loss": 2.7827,
+      "theoretical_loss": 3.6178217122677045,
+      "tokens_seen": 1095761920
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003072276338188008,
+      "loss": 2.7143,
+      "theoretical_loss": 3.6174171143714124,
+      "tokens_seen": 1097072640
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030699120484206546,
+      "loss": 2.6943,
+      "theoretical_loss": 3.617013134742405,
+      "tokens_seen": 1098383360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030675477586533006,
+      "loss": 2.7524,
+      "theoretical_loss": 3.616609771699806,
+      "tokens_seen": 1099694080
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030651834688859465,
+      "loss": 2.7242,
+      "theoretical_loss": 3.616207023569306,
+      "tokens_seen": 1101004800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030628191791185925,
+      "loss": 2.6719,
+      "theoretical_loss": 3.6158048886831287,
+      "tokens_seen": 1102315520
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003060454889351239,
+      "loss": 2.674,
+      "theoretical_loss": 3.6154033653799953,
+      "tokens_seen": 1103626240
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003058090599583885,
+      "loss": 2.6836,
+      "theoretical_loss": 3.615002452005096,
+      "tokens_seen": 1104936960
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003055726309816531,
+      "loss": 2.7231,
+      "theoretical_loss": 3.614602146910054,
+      "tokens_seen": 1106247680
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030533620200491775,
+      "loss": 2.7444,
+      "theoretical_loss": 3.614202448452893,
+      "tokens_seen": 1107558400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003050997730281823,
+      "loss": 2.7074,
+      "theoretical_loss": 3.613803354998007,
+      "tokens_seen": 1108869120
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030486334405144694,
+      "loss": 2.7037,
+      "theoretical_loss": 3.6134048649161237,
+      "tokens_seen": 1110179840
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003046269150747116,
+      "loss": 2.6998,
+      "theoretical_loss": 3.6130069765842787,
+      "tokens_seen": 1111490560
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030439048609797614,
+      "loss": 2.7138,
+      "theoretical_loss": 3.6126096883857777,
+      "tokens_seen": 1112801280
+    },
+    {
+      "epoch": 0.4,
+      "objective/train/docs_used": 617837,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.8926000595092773,
+      "objective/train/theoretical_loss": 3.61221299871017,
+      "objective/train/tokens_used": 1134572000,
+      "theoretical_loss": 3.61221299871017,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003041540571212408,
+      "loss": 2.7617,
+      "theoretical_loss": 3.61221299871017,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003039176281445054,
+      "loss": 2.7269,
+      "theoretical_loss": 3.611816905953212,
+      "tokens_seen": 1115422720
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030368119916777,
+      "loss": 2.6824,
+      "theoretical_loss": 3.6114214085168417,
+      "tokens_seen": 1116733440
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030344477019103464,
+      "loss": 2.7501,
+      "theoretical_loss": 3.611026504809142,
+      "tokens_seen": 1118044160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030320834121429923,
+      "loss": 2.712,
+      "theoretical_loss": 3.6106321932443137,
+      "tokens_seen": 1119354880
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030297191223756383,
+      "loss": 2.6591,
+      "theoretical_loss": 3.6102384722426435,
+      "tokens_seen": 1120665600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030273548326082843,
+      "loss": 2.7543,
+      "theoretical_loss": 3.609845340230474,
+      "tokens_seen": 1121976320
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003024990542840931,
+      "loss": 2.7203,
+      "theoretical_loss": 3.6094527956401734,
+      "tokens_seen": 1123287040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003022626253073577,
+      "loss": 2.7033,
+      "theoretical_loss": 3.6090608369101043,
+      "tokens_seen": 1124597760
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003020261963306223,
+      "loss": 2.6814,
+      "theoretical_loss": 3.6086694624845954,
+      "tokens_seen": 1125908480
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030178976735388693,
+      "loss": 2.669,
+      "theoretical_loss": 3.6082786708139114,
+      "tokens_seen": 1127219200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030155333837715147,
+      "loss": 2.6892,
+      "theoretical_loss": 3.6078884603542223,
+      "tokens_seen": 1128529920
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003013169094004161,
+      "loss": 2.6341,
+      "theoretical_loss": 3.607498829567576,
+      "tokens_seen": 1129840640
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003010804804236807,
+      "loss": 2.708,
+      "theoretical_loss": 3.6071097769218676,
+      "tokens_seen": 1131151360
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003008440514469453,
+      "loss": 2.7243,
+      "theoretical_loss": 3.606721300890811,
+      "tokens_seen": 1132462080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030060762247020997,
+      "loss": 2.7242,
+      "theoretical_loss": 3.6063333999539102,
+      "tokens_seen": 1133772800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00030037119349347457,
+      "loss": 2.6964,
+      "theoretical_loss": 3.6059460725964305,
+      "tokens_seen": 1135083520
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00030013476451673916,
+      "loss": 2.6908,
+      "theoretical_loss": 3.6055593173093694,
+      "tokens_seen": 1136394240
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029989833554000376,
+      "loss": 2.7061,
+      "theoretical_loss": 3.60517313258943,
+      "tokens_seen": 1137704960
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002996619065632684,
+      "loss": 2.7295,
+      "theoretical_loss": 3.604787516938992,
+      "tokens_seen": 1139015680
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000299425477586533,
+      "loss": 2.6895,
+      "theoretical_loss": 3.6044024688660823,
+      "tokens_seen": 1140326400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002991890486097976,
+      "loss": 2.7346,
+      "theoretical_loss": 3.604017986884349,
+      "tokens_seen": 1141637120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029895261963306226,
+      "loss": 2.6686,
+      "theoretical_loss": 3.603634069513035,
+      "tokens_seen": 1142947840
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002987161906563268,
+      "loss": 2.6908,
+      "theoretical_loss": 3.603250715276947,
+      "tokens_seen": 1144258560
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029847976167959146,
+      "loss": 2.6998,
+      "theoretical_loss": 3.6028679227064306,
+      "tokens_seen": 1145569280
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002982433327028561,
+      "loss": 2.75,
+      "theoretical_loss": 3.6024856903373434,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029800690372612065,
+      "loss": 2.7409,
+      "theoretical_loss": 3.602104016711027,
+      "tokens_seen": 1148190720
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002977704747493853,
+      "loss": 2.7675,
+      "theoretical_loss": 3.6017229003742806,
+      "tokens_seen": 1149501440
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002975340457726499,
+      "loss": 2.7389,
+      "theoretical_loss": 3.6013423398793347,
+      "tokens_seen": 1150812160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002972976167959145,
+      "loss": 2.7125,
+      "theoretical_loss": 3.600962333783824,
+      "tokens_seen": 1152122880
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029706118781917915,
+      "loss": 2.728,
+      "theoretical_loss": 3.6005828806507623,
+      "tokens_seen": 1153433600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029682475884244375,
+      "loss": 2.7785,
+      "theoretical_loss": 3.6002039790485147,
+      "tokens_seen": 1154744320
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029658832986570834,
+      "loss": 2.7339,
+      "theoretical_loss": 3.5998256275507723,
+      "tokens_seen": 1156055040
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029635190088897294,
+      "loss": 2.7564,
+      "theoretical_loss": 3.5994478247365276,
+      "tokens_seen": 1157365760
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002961154719122376,
+      "loss": 2.7336,
+      "theoretical_loss": 3.5990705691900464,
+      "tokens_seen": 1158676480
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029587904293550214,
+      "loss": 2.6968,
+      "theoretical_loss": 3.5986938595008455,
+      "tokens_seen": 1159987200
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002956426139587668,
+      "loss": 2.739,
+      "theoretical_loss": 3.5983176942636637,
+      "tokens_seen": 1161297920
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029540618498203144,
+      "loss": 2.7562,
+      "theoretical_loss": 3.597942072078439,
+      "tokens_seen": 1162608640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000295169756005296,
+      "loss": 2.6762,
+      "theoretical_loss": 3.5975669915502833,
+      "tokens_seen": 1163919360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029493332702856063,
+      "loss": 2.693,
+      "theoretical_loss": 3.597192451289457,
+      "tokens_seen": 1165230080
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029469689805182523,
+      "loss": 2.7433,
+      "theoretical_loss": 3.5968184499113436,
+      "tokens_seen": 1166540800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029446046907508983,
+      "loss": 2.7353,
+      "theoretical_loss": 3.5964449860364263,
+      "tokens_seen": 1167851520
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002942240400983545,
+      "loss": 2.7333,
+      "theoretical_loss": 3.5960720582902628,
+      "tokens_seen": 1169162240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002939876111216191,
+      "loss": 2.6914,
+      "theoretical_loss": 3.595699665303462,
+      "tokens_seen": 1170472960
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002937511821448837,
+      "loss": 2.6491,
+      "theoretical_loss": 3.595327805711657,
+      "tokens_seen": 1171783680
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002935147531681483,
+      "loss": 2.6961,
+      "theoretical_loss": 3.5949564781554857,
+      "tokens_seen": 1173094400
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002932783241914129,
+      "loss": 2.7129,
+      "theoretical_loss": 3.5945856812805626,
+      "tokens_seen": 1174405120
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002930418952146775,
+      "loss": 2.751,
+      "theoretical_loss": 3.594215413737457,
+      "tokens_seen": 1175715840
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002928054662379421,
+      "loss": 2.7487,
+      "theoretical_loss": 3.5938456741816682,
+      "tokens_seen": 1177026560
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029256903726120677,
+      "loss": 2.7021,
+      "theoretical_loss": 3.593476461273605,
+      "tokens_seen": 1178337280
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 651196,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.538501501083374,
+      "objective/train/theoretical_loss": 3.5931077736785584,
+      "objective/train/tokens_used": 1200108000,
+      "theoretical_loss": 3.5931077736785584,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002923326082844713,
+      "loss": 2.6904,
+      "theoretical_loss": 3.5931077736785584,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029209617930773597,
+      "loss": 2.7642,
+      "theoretical_loss": 3.5927396100666806,
+      "tokens_seen": 1180958720
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002918597503310006,
+      "loss": 2.7317,
+      "theoretical_loss": 3.592371969112962,
+      "tokens_seen": 1182269440
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029162332135426516,
+      "loss": 2.6943,
+      "theoretical_loss": 3.5920048494972074,
+      "tokens_seen": 1183580160
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002913868923775298,
+      "loss": 2.7779,
+      "theoretical_loss": 3.5916382499040136,
+      "tokens_seen": 1184890880
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002911504634007944,
+      "loss": 2.7194,
+      "theoretical_loss": 3.591272169022747,
+      "tokens_seen": 1186201600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000290914034424059,
+      "loss": 2.6754,
+      "theoretical_loss": 3.590906605547521,
+      "tokens_seen": 1187512320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029067760544732366,
+      "loss": 2.7506,
+      "theoretical_loss": 3.590541558177172,
+      "tokens_seen": 1188823040
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029044117647058826,
+      "loss": 2.7132,
+      "theoretical_loss": 3.5901770256152403,
+      "tokens_seen": 1190133760
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029020474749385285,
+      "loss": 2.6936,
+      "theoretical_loss": 3.589813006569945,
+      "tokens_seen": 1191444480
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028996831851711745,
+      "loss": 2.7217,
+      "theoretical_loss": 3.589449499754164,
+      "tokens_seen": 1192755200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002897318895403821,
+      "loss": 2.7581,
+      "theoretical_loss": 3.5890865038854103,
+      "tokens_seen": 1194065920
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028949546056364665,
+      "loss": 2.7031,
+      "theoretical_loss": 3.5887240176858115,
+      "tokens_seen": 1195376640
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002892590315869113,
+      "loss": 2.7189,
+      "theoretical_loss": 3.588362039882088,
+      "tokens_seen": 1196687360
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028902260261017595,
+      "loss": 2.7182,
+      "theoretical_loss": 3.5880005692055317,
+      "tokens_seen": 1197998080
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002887861736334405,
+      "loss": 2.6783,
+      "theoretical_loss": 3.5876396043919834,
+      "tokens_seen": 1199308800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028854974465670515,
+      "loss": 2.7205,
+      "theoretical_loss": 3.587279144181813,
+      "tokens_seen": 1200619520
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028831331567996974,
+      "loss": 2.7329,
+      "theoretical_loss": 3.586919187319897,
+      "tokens_seen": 1201930240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028807688670323434,
+      "loss": 2.6913,
+      "theoretical_loss": 3.5865597325555987,
+      "tokens_seen": 1203240960
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000287840457726499,
+      "loss": 2.7401,
+      "theoretical_loss": 3.5862007786427474,
+      "tokens_seen": 1204551680
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002876040287497636,
+      "loss": 2.7276,
+      "theoretical_loss": 3.585842324339616,
+      "tokens_seen": 1205862400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002873675997730282,
+      "loss": 2.7041,
+      "theoretical_loss": 3.5854843684089017,
+      "tokens_seen": 1207173120
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002871311707962928,
+      "loss": 2.7244,
+      "theoretical_loss": 3.5851269096177054,
+      "tokens_seen": 1208483840
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028689474181955744,
+      "loss": 2.721,
+      "theoretical_loss": 3.5847699467375103,
+      "tokens_seen": 1209794560
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028665831284282203,
+      "loss": 2.7419,
+      "theoretical_loss": 3.584413478544163,
+      "tokens_seen": 1211105280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028642188386608663,
+      "loss": 2.729,
+      "theoretical_loss": 3.584057503817852,
+      "tokens_seen": 1212416000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002861854548893513,
+      "loss": 2.6775,
+      "theoretical_loss": 3.5837020213430884,
+      "tokens_seen": 1213726720
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002859490259126158,
+      "loss": 2.7225,
+      "theoretical_loss": 3.583347029908686,
+      "tokens_seen": 1215037440
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002857125969358805,
+      "loss": 2.7179,
+      "theoretical_loss": 3.582992528307741,
+      "tokens_seen": 1216348160
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028547616795914513,
+      "loss": 2.7152,
+      "theoretical_loss": 3.5826385153376124,
+      "tokens_seen": 1217658880
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028523973898240967,
+      "loss": 2.7084,
+      "theoretical_loss": 3.582284989799902,
+      "tokens_seen": 1218969600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002850033100056743,
+      "loss": 2.7369,
+      "theoretical_loss": 3.5819319505004366,
+      "tokens_seen": 1220280320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002847668810289389,
+      "loss": 2.7269,
+      "theoretical_loss": 3.581579396249245,
+      "tokens_seen": 1221591040
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002845304520522035,
+      "loss": 2.7058,
+      "theoretical_loss": 3.5812273258605436,
+      "tokens_seen": 1222901760
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028429402307546817,
+      "loss": 2.7345,
+      "theoretical_loss": 3.580875738152713,
+      "tokens_seen": 1224212480
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002840575940987327,
+      "loss": 2.7566,
+      "theoretical_loss": 3.5805246319482817,
+      "tokens_seen": 1225523200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028382116512199737,
+      "loss": 2.7297,
+      "theoretical_loss": 3.5801740060739036,
+      "tokens_seen": 1226833920
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028358473614526196,
+      "loss": 2.6699,
+      "theoretical_loss": 3.579823859360345,
+      "tokens_seen": 1228144640
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028334830716852656,
+      "loss": 2.7415,
+      "theoretical_loss": 3.5794741906424594,
+      "tokens_seen": 1229455360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028311187819179116,
+      "loss": 2.7704,
+      "theoretical_loss": 3.5791249987591733,
+      "tokens_seen": 1230766080
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002828754492150558,
+      "loss": 2.696,
+      "theoretical_loss": 3.5787762825534664,
+      "tokens_seen": 1232076800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002826390202383204,
+      "loss": 2.7599,
+      "theoretical_loss": 3.578428040872351,
+      "tokens_seen": 1233387520
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.000282402591261585,
+      "loss": 2.719,
+      "theoretical_loss": 3.5780802725668575,
+      "tokens_seen": 1234698240
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028216616228484966,
+      "loss": 2.7349,
+      "theoretical_loss": 3.5777329764920136,
+      "tokens_seen": 1236008960
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002819297333081142,
+      "loss": 2.7247,
+      "theoretical_loss": 3.5773861515068273,
+      "tokens_seen": 1237319680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028169330433137885,
+      "loss": 2.7526,
+      "theoretical_loss": 3.5770397964742675,
+      "tokens_seen": 1238630400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002814568753546435,
+      "loss": 2.6973,
+      "theoretical_loss": 3.5766939102612474,
+      "tokens_seen": 1239941120
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028122044637790805,
+      "loss": 2.7114,
+      "theoretical_loss": 3.5763484917386057,
+      "tokens_seen": 1241251840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002809840174011727,
+      "loss": 2.7062,
+      "theoretical_loss": 3.576003539781091,
+      "tokens_seen": 1242562560
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002807475884244373,
+      "loss": 2.7452,
+      "theoretical_loss": 3.5756590532673407,
+      "tokens_seen": 1243873280
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 687081,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7596964836120605,
+      "objective/train/theoretical_loss": 3.5753150310798665,
+      "objective/train/tokens_used": 1265644000,
+      "theoretical_loss": 3.5753150310798665,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002805111594477019,
+      "loss": 2.7577,
+      "theoretical_loss": 3.5753150310798665,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028027473047096654,
+      "loss": 2.7681,
+      "theoretical_loss": 3.5749714721050343,
+      "tokens_seen": 1246494720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028003830149423114,
+      "loss": 2.7865,
+      "theoretical_loss": 3.5746283752330505,
+      "tokens_seen": 1247805440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027980187251749574,
+      "loss": 2.7529,
+      "theoretical_loss": 3.574285739357941,
+      "tokens_seen": 1249116160
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027956544354076034,
+      "loss": 2.8135,
+      "theoretical_loss": 3.5739435633775365,
+      "tokens_seen": 1250426880
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.000279329014564025,
+      "loss": 2.7906,
+      "theoretical_loss": 3.573601846193453,
+      "tokens_seen": 1251737600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002790925855872896,
+      "loss": 2.7711,
+      "theoretical_loss": 3.573260586711079,
+      "tokens_seen": 1253048320
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002788561566105542,
+      "loss": 2.7497,
+      "theoretical_loss": 3.5729197838395543,
+      "tokens_seen": 1254359040
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027861972763381884,
+      "loss": 2.746,
+      "theoretical_loss": 3.5725794364917554,
+      "tokens_seen": 1255669760
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002783832986570834,
+      "loss": 2.7584,
+      "theoretical_loss": 3.5722395435842778,
+      "tokens_seen": 1256980480
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027814686968034803,
+      "loss": 2.6996,
+      "theoretical_loss": 3.5719001040374216,
+      "tokens_seen": 1258291200
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027791044070361263,
+      "loss": 2.7221,
+      "theoretical_loss": 3.5715611167751717,
+      "tokens_seen": 1259601920
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002776740117268772,
+      "loss": 2.7164,
+      "theoretical_loss": 3.571222580725184,
+      "tokens_seen": 1260912640
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002774375827501419,
+      "loss": 2.6775,
+      "theoretical_loss": 3.5708844948187686,
+      "tokens_seen": 1262223360
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002772011537734065,
+      "loss": 2.7207,
+      "theoretical_loss": 3.5705468579908715,
+      "tokens_seen": 1263534080
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027696472479667107,
+      "loss": 2.7102,
+      "theoretical_loss": 3.5702096691800627,
+      "tokens_seen": 1264844800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027672829581993567,
+      "loss": 2.7278,
+      "theoretical_loss": 3.5698729273285146,
+      "tokens_seen": 1266155520
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002764918668432003,
+      "loss": 2.6994,
+      "theoretical_loss": 3.5695366313819914,
+      "tokens_seen": 1267466240
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002762554378664649,
+      "loss": 2.6967,
+      "theoretical_loss": 3.56920078028983,
+      "tokens_seen": 1268776960
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002760190088897295,
+      "loss": 2.6664,
+      "theoretical_loss": 3.5688653730049245,
+      "tokens_seen": 1270087680
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027578257991299417,
+      "loss": 2.6885,
+      "theoretical_loss": 3.5685304084837126,
+      "tokens_seen": 1271398400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002755461509362587,
+      "loss": 2.7197,
+      "theoretical_loss": 3.5681958856861558,
+      "tokens_seen": 1272709120
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027530972195952336,
+      "loss": 2.7167,
+      "theoretical_loss": 3.5678618035757297,
+      "tokens_seen": 1274019840
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000275073292982788,
+      "loss": 2.695,
+      "theoretical_loss": 3.5675281611194025,
+      "tokens_seen": 1275330560
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027483686400605256,
+      "loss": 2.711,
+      "theoretical_loss": 3.5671949572876245,
+      "tokens_seen": 1276641280
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002746004350293172,
+      "loss": 2.6874,
+      "theoretical_loss": 3.5668621910543097,
+      "tokens_seen": 1277952000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002743640060525818,
+      "loss": 2.644,
+      "theoretical_loss": 3.566529861396822,
+      "tokens_seen": 1279262720
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002741275770758464,
+      "loss": 2.6857,
+      "theoretical_loss": 3.566197967295959,
+      "tokens_seen": 1280573440
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027389114809911106,
+      "loss": 2.6703,
+      "theoretical_loss": 3.56586650773594,
+      "tokens_seen": 1281884160
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027365471912237565,
+      "loss": 2.6677,
+      "theoretical_loss": 3.5655354817043863,
+      "tokens_seen": 1283194880
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027341829014564025,
+      "loss": 2.7047,
+      "theoretical_loss": 3.5652048881923095,
+      "tokens_seen": 1284505600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027318186116890485,
+      "loss": 2.6536,
+      "theoretical_loss": 3.5648747261940974,
+      "tokens_seen": 1285816320
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002729454321921695,
+      "loss": 2.6367,
+      "theoretical_loss": 3.5645449947074956,
+      "tokens_seen": 1287127040
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027270900321543404,
+      "loss": 2.6546,
+      "theoretical_loss": 3.5642156927335975,
+      "tokens_seen": 1288437760
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002724725742386987,
+      "loss": 2.6724,
+      "theoretical_loss": 3.563886819276826,
+      "tokens_seen": 1289748480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027223614526196335,
+      "loss": 2.6625,
+      "theoretical_loss": 3.5635583733449203,
+      "tokens_seen": 1291059200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002719997162852279,
+      "loss": 2.6636,
+      "theoretical_loss": 3.5632303539489225,
+      "tokens_seen": 1292369920
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027176328730849254,
+      "loss": 2.6695,
+      "theoretical_loss": 3.562902760103161,
+      "tokens_seen": 1293680640
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027152685833175714,
+      "loss": 2.6111,
+      "theoretical_loss": 3.56257559082524,
+      "tokens_seen": 1294991360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027129042935502174,
+      "loss": 2.7303,
+      "theoretical_loss": 3.562248845136021,
+      "tokens_seen": 1296302080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002710540003782864,
+      "loss": 2.6753,
+      "theoretical_loss": 3.5619225220596116,
+      "tokens_seen": 1297612800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000270817571401551,
+      "loss": 2.7305,
+      "theoretical_loss": 3.56159662062335,
+      "tokens_seen": 1298923520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002705811424248156,
+      "loss": 2.6723,
+      "theoretical_loss": 3.5612711398577916,
+      "tokens_seen": 1300234240
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002703447134480802,
+      "loss": 2.6636,
+      "theoretical_loss": 3.5609460787966962,
+      "tokens_seen": 1301544960
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00027010828447134483,
+      "loss": 2.6819,
+      "theoretical_loss": 3.560621436477012,
+      "tokens_seen": 1302855680
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026987185549460943,
+      "loss": 2.6488,
+      "theoretical_loss": 3.560297211938863,
+      "tokens_seen": 1304166400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026963542651787403,
+      "loss": 2.6952,
+      "theoretical_loss": 3.5599734042255373,
+      "tokens_seen": 1305477120
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002693989975411387,
+      "loss": 2.693,
+      "theoretical_loss": 3.559650012383469,
+      "tokens_seen": 1306787840
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002691625685644032,
+      "loss": 2.6732,
+      "theoretical_loss": 3.559327035462228,
+      "tokens_seen": 1308098560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002689261395876679,
+      "loss": 2.6645,
+      "theoretical_loss": 3.5590044725145074,
+      "tokens_seen": 1309409280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5616705212962695,
+      "debugging/distinct-1-grams": 0.796841979490702,
+      "debugging/distinct-2-grams": 0.9743537545539217,
+      "debugging/entropy-1-grams": 6.281224573287751,
+      "debugging/entropy-2-grams": 7.318801977908639,
+      "debugging/length": 460.25,
+      "debugging/num_segments": 24,
+      "epoch": 0.47,
+      "objective/train/docs_used": 722210,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.034902572631836,
+      "objective/train/theoretical_loss": 3.5586823225961064,
+      "objective/train/tokens_used": 1331180000,
+      "theoretical_loss": 3.5586823225961064,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002686897106109325,
+      "loss": 2.6237,
+      "theoretical_loss": 3.5586823225961064,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026845328163419707,
+      "loss": 2.6581,
+      "theoretical_loss": 3.5583605847659214,
+      "tokens_seen": 1312030720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002682168526574617,
+      "loss": 2.6362,
+      "theoretical_loss": 3.558039258085929,
+      "tokens_seen": 1313341440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002679804236807263,
+      "loss": 2.7213,
+      "theoretical_loss": 3.5577183416211744,
+      "tokens_seen": 1314652160
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002677439947039909,
+      "loss": 2.6685,
+      "theoretical_loss": 3.5573978344397594,
+      "tokens_seen": 1315962880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026750756572725557,
+      "loss": 2.6144,
+      "theoretical_loss": 3.5570777356128276,
+      "tokens_seen": 1317273600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026727113675052016,
+      "loss": 2.6287,
+      "theoretical_loss": 3.556758044214552,
+      "tokens_seen": 1318584320
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026703470777378476,
+      "loss": 2.6641,
+      "theoretical_loss": 3.5564387593221225,
+      "tokens_seen": 1319895040
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026679827879704936,
+      "loss": 2.6217,
+      "theoretical_loss": 3.556119880015733,
+      "tokens_seen": 1321205760
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.000266561849820314,
+      "loss": 2.6583,
+      "theoretical_loss": 3.5558014053785687,
+      "tokens_seen": 1322516480
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026632542084357855,
+      "loss": 2.647,
+      "theoretical_loss": 3.5554833344967927,
+      "tokens_seen": 1323827200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002660889918668432,
+      "loss": 2.6288,
+      "theoretical_loss": 3.5551656664595344,
+      "tokens_seen": 1325137920
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026585256289010786,
+      "loss": 2.619,
+      "theoretical_loss": 3.5548484003588756,
+      "tokens_seen": 1326448640
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002656161339133724,
+      "loss": 2.6547,
+      "theoretical_loss": 3.554531535289841,
+      "tokens_seen": 1327759360
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026537970493663705,
+      "loss": 2.678,
+      "theoretical_loss": 3.5542150703503808,
+      "tokens_seen": 1329070080
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026514327595990165,
+      "loss": 2.6858,
+      "theoretical_loss": 3.553899004641363,
+      "tokens_seen": 1330380800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026490684698316625,
+      "loss": 2.6943,
+      "theoretical_loss": 3.55358333726656,
+      "tokens_seen": 1331691520
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002646704180064309,
+      "loss": 2.6441,
+      "theoretical_loss": 3.553268067332634,
+      "tokens_seen": 1333002240
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002644339890296955,
+      "loss": 2.679,
+      "theoretical_loss": 3.5529531939491275,
+      "tokens_seen": 1334312960
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002641975600529601,
+      "loss": 2.6601,
+      "theoretical_loss": 3.552638716228449,
+      "tokens_seen": 1335623680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002639611310762247,
+      "loss": 2.6693,
+      "theoretical_loss": 3.552324633285865,
+      "tokens_seen": 1336934400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026372470209948934,
+      "loss": 2.6284,
+      "theoretical_loss": 3.5520109442394827,
+      "tokens_seen": 1338245120
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026348827312275394,
+      "loss": 2.6787,
+      "theoretical_loss": 3.5516976482102414,
+      "tokens_seen": 1339555840
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026325184414601854,
+      "loss": 2.7308,
+      "theoretical_loss": 3.5513847443218998,
+      "tokens_seen": 1340866560
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002630154151692832,
+      "loss": 2.6488,
+      "theoretical_loss": 3.551072231701025,
+      "tokens_seen": 1342177280
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026277898619254773,
+      "loss": 2.6368,
+      "theoretical_loss": 3.550760109476979,
+      "tokens_seen": 1343488000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002625425572158124,
+      "loss": 2.659,
+      "theoretical_loss": 3.550448376781909,
+      "tokens_seen": 1344798720
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026230612823907704,
+      "loss": 2.6445,
+      "theoretical_loss": 3.550137032750734,
+      "tokens_seen": 1346109440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002620696992623416,
+      "loss": 2.6342,
+      "theoretical_loss": 3.5498260765211356,
+      "tokens_seen": 1347420160
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026183327028560623,
+      "loss": 2.697,
+      "theoretical_loss": 3.549515507233543,
+      "tokens_seen": 1348730880
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026159684130887083,
+      "loss": 2.7078,
+      "theoretical_loss": 3.5492053240311257,
+      "tokens_seen": 1350041600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002613604123321354,
+      "loss": 2.6624,
+      "theoretical_loss": 3.5488955260597783,
+      "tokens_seen": 1351352320
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002611239833554001,
+      "loss": 2.7078,
+      "theoretical_loss": 3.5485861124681124,
+      "tokens_seen": 1352663040
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002608875543786647,
+      "loss": 2.6611,
+      "theoretical_loss": 3.5482770824074428,
+      "tokens_seen": 1353973760
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002606511254019293,
+      "loss": 2.6832,
+      "theoretical_loss": 3.5479684350317777,
+      "tokens_seen": 1355284480
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026041469642519387,
+      "loss": 2.7189,
+      "theoretical_loss": 3.5476601694978083,
+      "tokens_seen": 1356595200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026017826744845847,
+      "loss": 2.6972,
+      "theoretical_loss": 3.5473522849648944,
+      "tokens_seen": 1357905920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025994183847172307,
+      "loss": 2.6848,
+      "theoretical_loss": 3.547044780595058,
+      "tokens_seen": 1359216640
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002597054094949877,
+      "loss": 2.6864,
+      "theoretical_loss": 3.5467376555529686,
+      "tokens_seen": 1360527360
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002594689805182523,
+      "loss": 2.6594,
+      "theoretical_loss": 3.546430909005934,
+      "tokens_seen": 1361838080
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002592325515415169,
+      "loss": 2.6559,
+      "theoretical_loss": 3.5461245401238894,
+      "tokens_seen": 1363148800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025899612256478156,
+      "loss": 2.6862,
+      "theoretical_loss": 3.545818548079385,
+      "tokens_seen": 1364459520
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002587596935880461,
+      "loss": 2.6839,
+      "theoretical_loss": 3.545512932047579,
+      "tokens_seen": 1365770240
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025852326461131076,
+      "loss": 2.667,
+      "theoretical_loss": 3.5452076912062216,
+      "tokens_seen": 1367080960
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002582868356345754,
+      "loss": 2.7063,
+      "theoretical_loss": 3.54490282473565,
+      "tokens_seen": 1368391680
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025805040665783995,
+      "loss": 2.6548,
+      "theoretical_loss": 3.544598331818772,
+      "tokens_seen": 1369702400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002578139776811046,
+      "loss": 2.6861,
+      "theoretical_loss": 3.5442942116410614,
+      "tokens_seen": 1371013120
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002575775487043692,
+      "loss": 2.657,
+      "theoretical_loss": 3.5439904633905424,
+      "tokens_seen": 1372323840
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002573411197276338,
+      "loss": 2.6873,
+      "theoretical_loss": 3.5436870862577825,
+      "tokens_seen": 1373634560
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025710469075089845,
+      "loss": 2.6745,
+      "theoretical_loss": 3.5433840794358806,
+      "tokens_seen": 1374945280
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 759843,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5367300510406494,
+      "objective/train/theoretical_loss": 3.5430814421204575,
+      "objective/train/tokens_used": 1396716000,
+      "theoretical_loss": 3.5430814421204575,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025686826177416305,
+      "loss": 2.6763,
+      "theoretical_loss": 3.5430814421204575,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025663183279742765,
+      "loss": 2.7077,
+      "theoretical_loss": 3.5427791735096448,
+      "tokens_seen": 1377566720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025639540382069225,
+      "loss": 2.6516,
+      "theoretical_loss": 3.542477272804075,
+      "tokens_seen": 1378877440
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002561589748439569,
+      "loss": 2.6722,
+      "theoretical_loss": 3.542175739206872,
+      "tokens_seen": 1380188160
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002559225458672215,
+      "loss": 2.6098,
+      "theoretical_loss": 3.541874571923639,
+      "tokens_seen": 1381498880
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002556861168904861,
+      "loss": 2.6425,
+      "theoretical_loss": 3.5415737701624517,
+      "tokens_seen": 1382809600
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025544968791375074,
+      "loss": 2.6648,
+      "theoretical_loss": 3.5412733331338453,
+      "tokens_seen": 1384120320
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002552132589370153,
+      "loss": 2.6946,
+      "theoretical_loss": 3.5409732600508055,
+      "tokens_seen": 1385431040
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025497682996027994,
+      "loss": 2.6316,
+      "theoretical_loss": 3.5406735501287594,
+      "tokens_seen": 1386741760
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025474040098354454,
+      "loss": 2.6562,
+      "theoretical_loss": 3.5403742025855647,
+      "tokens_seen": 1388052480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025450397200680913,
+      "loss": 2.6769,
+      "theoretical_loss": 3.5400752166415,
+      "tokens_seen": 1389363200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002542675430300738,
+      "loss": 2.639,
+      "theoretical_loss": 3.5397765915192547,
+      "tokens_seen": 1390673920
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002540311140533384,
+      "loss": 2.686,
+      "theoretical_loss": 3.5394783264439207,
+      "tokens_seen": 1391984640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000253794685076603,
+      "loss": 2.6641,
+      "theoretical_loss": 3.5391804206429818,
+      "tokens_seen": 1393295360
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002535582560998676,
+      "loss": 2.7026,
+      "theoretical_loss": 3.538882873346303,
+      "tokens_seen": 1394606080
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025332182712313223,
+      "loss": 2.7189,
+      "theoretical_loss": 3.5385856837861223,
+      "tokens_seen": 1395916800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002530853981463968,
+      "loss": 2.7667,
+      "theoretical_loss": 3.538288851197042,
+      "tokens_seen": 1397227520
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002528489691696614,
+      "loss": 2.7162,
+      "theoretical_loss": 3.537992374816016,
+      "tokens_seen": 1398538240
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002526125401929261,
+      "loss": 2.6978,
+      "theoretical_loss": 3.537696253882345,
+      "tokens_seen": 1399848960
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002523761112161906,
+      "loss": 2.6824,
+      "theoretical_loss": 3.5374004876376617,
+      "tokens_seen": 1401159680
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025213968223945527,
+      "loss": 2.7069,
+      "theoretical_loss": 3.5371050753259268,
+      "tokens_seen": 1402470400
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002519032532627199,
+      "loss": 2.664,
+      "theoretical_loss": 3.5368100161934155,
+      "tokens_seen": 1403781120
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025166682428598447,
+      "loss": 2.6769,
+      "theoretical_loss": 3.5365153094887107,
+      "tokens_seen": 1405091840
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002514303953092491,
+      "loss": 2.7123,
+      "theoretical_loss": 3.536220954462693,
+      "tokens_seen": 1406402560
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002511939663325137,
+      "loss": 2.7079,
+      "theoretical_loss": 3.535926950368532,
+      "tokens_seen": 1407713280
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002509575373557783,
+      "loss": 2.709,
+      "theoretical_loss": 3.5356332964616755,
+      "tokens_seen": 1409024000
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025072110837904296,
+      "loss": 2.6828,
+      "theoretical_loss": 3.535339991999842,
+      "tokens_seen": 1410334720
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025048467940230756,
+      "loss": 2.674,
+      "theoretical_loss": 3.5350470362430126,
+      "tokens_seen": 1411645440
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025024825042557216,
+      "loss": 2.6525,
+      "theoretical_loss": 3.5347544284534207,
+      "tokens_seen": 1412956160
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00025001182144883676,
+      "loss": 2.6333,
+      "theoretical_loss": 3.534462167895541,
+      "tokens_seen": 1414266880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024977539247210135,
+      "loss": 2.7273,
+      "theoretical_loss": 3.534170253836085,
+      "tokens_seen": 1415577600
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.000249538963495366,
+      "loss": 2.6904,
+      "theoretical_loss": 3.533878685543989,
+      "tokens_seen": 1416888320
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002493025345186306,
+      "loss": 2.686,
+      "theoretical_loss": 3.5335874622904067,
+      "tokens_seen": 1418199040
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002490661055418952,
+      "loss": 2.6896,
+      "theoretical_loss": 3.5332965833486987,
+      "tokens_seen": 1419509760
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024882967656515985,
+      "loss": 2.639,
+      "theoretical_loss": 3.5330060479944265,
+      "tokens_seen": 1420820480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024859324758842445,
+      "loss": 2.6443,
+      "theoretical_loss": 3.5327158555053413,
+      "tokens_seen": 1422131200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024835681861168905,
+      "loss": 2.6638,
+      "theoretical_loss": 3.532426005161377,
+      "tokens_seen": 1423441920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002481203896349537,
+      "loss": 2.6827,
+      "theoretical_loss": 3.5321364962446418,
+      "tokens_seen": 1424752640
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002478839606582183,
+      "loss": 2.6186,
+      "theoretical_loss": 3.5318473280394063,
+      "tokens_seen": 1426063360
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002476475316814829,
+      "loss": 2.6693,
+      "theoretical_loss": 3.5315584998321,
+      "tokens_seen": 1427374080
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002474111027047475,
+      "loss": 2.6997,
+      "theoretical_loss": 3.5312700109112996,
+      "tokens_seen": 1428684800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002471746737280121,
+      "loss": 2.6521,
+      "theoretical_loss": 3.530981860567721,
+      "tokens_seen": 1429995520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024693824475127674,
+      "loss": 2.6683,
+      "theoretical_loss": 3.530694048094211,
+      "tokens_seen": 1431306240
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024670181577454134,
+      "loss": 2.7062,
+      "theoretical_loss": 3.53040657278574,
+      "tokens_seen": 1432616960
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024646538679780594,
+      "loss": 2.6659,
+      "theoretical_loss": 3.530119433939393,
+      "tokens_seen": 1433927680
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024622895782107053,
+      "loss": 2.6449,
+      "theoretical_loss": 3.52983263085436,
+      "tokens_seen": 1435238400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024600435029317194,
+      "loss": 2.6295,
+      "theoretical_loss": 3.529546162831931,
+      "tokens_seen": 1436549120
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024576792131643653,
+      "loss": 2.6496,
+      "theoretical_loss": 3.5292600291754836,
+      "tokens_seen": 1437859840
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00024553149233970113,
+      "loss": 2.6996,
+      "theoretical_loss": 3.5289742291904784,
+      "tokens_seen": 1439170560
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002452950633629658,
+      "loss": 2.6579,
+      "theoretical_loss": 3.5286887621844496,
+      "tokens_seen": 1440481280
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 796239,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.210075855255127,
+      "objective/train/theoretical_loss": 3.5284036274669957,
+      "objective/train/tokens_used": 1462252000,
+      "theoretical_loss": 3.5284036274669957,
+      "tokens_seen": 1441792000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0002450586343862304,
+      "loss": 2.6158,
+      "theoretical_loss": 3.5284036274669957,
+      "tokens_seen": 1441792000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.000244822205409495,
+      "loss": 2.6155,
+      "theoretical_loss": 3.5281188243497743,
+      "tokens_seen": 1443102720
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024458577643275963,
+      "loss": 2.6979,
+      "theoretical_loss": 3.527834352146492,
+      "tokens_seen": 1444413440
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002443493474560242,
+      "loss": 2.5931,
+      "theoretical_loss": 3.527550210172895,
+      "tokens_seen": 1445724160
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024411291847928882,
+      "loss": 2.6179,
+      "theoretical_loss": 3.5272663977467658,
+      "tokens_seen": 1447034880
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024387648950255345,
+      "loss": 2.5787,
+      "theoretical_loss": 3.5269829141879114,
+      "tokens_seen": 1448345600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024364006052581805,
+      "loss": 2.6429,
+      "theoretical_loss": 3.5266997588181557,
+      "tokens_seen": 1449656320
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024340363154908264,
+      "loss": 2.663,
+      "theoretical_loss": 3.5264169309613353,
+      "tokens_seen": 1450967040
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002431672025723473,
+      "loss": 2.6679,
+      "theoretical_loss": 3.526134429943286,
+      "tokens_seen": 1452277760
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002429307735956119,
+      "loss": 2.6313,
+      "theoretical_loss": 3.5258522550918414,
+      "tokens_seen": 1453588480
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002426943446188765,
+      "loss": 2.6554,
+      "theoretical_loss": 3.525570405736818,
+      "tokens_seen": 1454899200
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024245791564214111,
+      "loss": 2.6154,
+      "theoretical_loss": 3.525288881210015,
+      "tokens_seen": 1456209920
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002422214866654057,
+      "loss": 2.5956,
+      "theoretical_loss": 3.525007680845202,
+      "tokens_seen": 1457520640
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024198505768867034,
+      "loss": 2.6826,
+      "theoretical_loss": 3.524726803978112,
+      "tokens_seen": 1458831360
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024174862871193496,
+      "loss": 2.6628,
+      "theoretical_loss": 3.5244462499464353,
+      "tokens_seen": 1460142080
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024151219973519956,
+      "loss": 2.7094,
+      "theoretical_loss": 3.5241660180898107,
+      "tokens_seen": 1461452800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024127577075846416,
+      "loss": 2.5873,
+      "theoretical_loss": 3.5238861077498185,
+      "tokens_seen": 1462763520
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024103934178172875,
+      "loss": 2.6598,
+      "theoretical_loss": 3.5236065182699736,
+      "tokens_seen": 1464074240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024080291280499338,
+      "loss": 2.644,
+      "theoretical_loss": 3.523327248995716,
+      "tokens_seen": 1465384960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.000240566483828258,
+      "loss": 2.6299,
+      "theoretical_loss": 3.523048299274407,
+      "tokens_seen": 1466695680
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.0002403300548515226,
+      "loss": 2.6554,
+      "theoretical_loss": 3.52276966845532,
+      "tokens_seen": 1468006400
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00024009362587478722,
+      "loss": 2.6341,
+      "theoretical_loss": 3.5224913558896285,
+      "tokens_seen": 1469317120
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023985719689805182,
+      "loss": 2.6504,
+      "theoretical_loss": 3.52221336093041,
+      "tokens_seen": 1470627840
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023962076792131642,
+      "loss": 2.6344,
+      "theoretical_loss": 3.5219356829326265,
+      "tokens_seen": 1471938560
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023938433894458107,
+      "loss": 2.6389,
+      "theoretical_loss": 3.5216583212531267,
+      "tokens_seen": 1473249280
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023914790996784567,
+      "loss": 2.5684,
+      "theoretical_loss": 3.521381275250633,
+      "tokens_seen": 1474560000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023891148099111027,
+      "loss": 2.6217,
+      "theoretical_loss": 3.5211045442857367,
+      "tokens_seen": 1475870720
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002386750520143749,
+      "loss": 2.6209,
+      "theoretical_loss": 3.5208281277208915,
+      "tokens_seen": 1477181440
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002384386230376395,
+      "loss": 2.5985,
+      "theoretical_loss": 3.5205520249204048,
+      "tokens_seen": 1478492160
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023820219406090409,
+      "loss": 2.5928,
+      "theoretical_loss": 3.5202762352504307,
+      "tokens_seen": 1479802880
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023796576508416874,
+      "loss": 2.5653,
+      "theoretical_loss": 3.5200007580789663,
+      "tokens_seen": 1481113600
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023772933610743334,
+      "loss": 2.6427,
+      "theoretical_loss": 3.519725592775839,
+      "tokens_seen": 1482424320
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023749290713069793,
+      "loss": 2.5693,
+      "theoretical_loss": 3.5194507387127048,
+      "tokens_seen": 1483735040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023725647815396256,
+      "loss": 2.5842,
+      "theoretical_loss": 3.5191761952630385,
+      "tokens_seen": 1485045760
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023702004917722715,
+      "loss": 2.62,
+      "theoretical_loss": 3.518901961802128,
+      "tokens_seen": 1486356480
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023678362020049178,
+      "loss": 2.554,
+      "theoretical_loss": 3.518628037707067,
+      "tokens_seen": 1487667200
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002365471912237564,
+      "loss": 2.5887,
+      "theoretical_loss": 3.518354422356748,
+      "tokens_seen": 1488977920
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.000236310762247021,
+      "loss": 2.624,
+      "theoretical_loss": 3.5180811151318556,
+      "tokens_seen": 1490288640
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0002360743332702856,
+      "loss": 2.6393,
+      "theoretical_loss": 3.5178081154148613,
+      "tokens_seen": 1491599360
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023583790429355022,
+      "loss": 2.5498,
+      "theoretical_loss": 3.5175354225900146,
+      "tokens_seen": 1492910080
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023560147531681482,
+      "loss": 2.6536,
+      "theoretical_loss": 3.517263036043337,
+      "tokens_seen": 1494220800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023536504634007945,
+      "loss": 2.6056,
+      "theoretical_loss": 3.5169909551626155,
+      "tokens_seen": 1495531520
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00023512861736334407,
+      "loss": 2.5438,
+      "theoretical_loss": 3.5167191793373966,
+      "tokens_seen": 1496842240
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023489218838660867,
+      "loss": 2.6232,
+      "theoretical_loss": 3.516447707958979,
+      "tokens_seen": 1498152960
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023465575940987326,
+      "loss": 2.6078,
+      "theoretical_loss": 3.516176540420406,
+      "tokens_seen": 1499463680
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002344193304331379,
+      "loss": 2.6309,
+      "theoretical_loss": 3.5159056761164624,
+      "tokens_seen": 1500774400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023418290145640251,
+      "loss": 2.6253,
+      "theoretical_loss": 3.515635114443664,
+      "tokens_seen": 1502085120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002339464724796671,
+      "loss": 2.6306,
+      "theoretical_loss": 3.5153648548002536,
+      "tokens_seen": 1503395840
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023371004350293174,
+      "loss": 2.5992,
+      "theoretical_loss": 3.5150948965861932,
+      "tokens_seen": 1504706560
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002334854359750331,
+      "loss": 2.6001,
+      "theoretical_loss": 3.5148252392031587,
+      "tokens_seen": 1506017280
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 833070,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5959901809692383,
+      "objective/train/theoretical_loss": 3.5145558820545335,
+      "objective/train/tokens_used": 1527788000,
+      "theoretical_loss": 3.5145558820545335,
+      "tokens_seen": 1507328000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002332490069982977,
+      "loss": 2.5761,
+      "theoretical_loss": 3.5145558820545335,
+      "tokens_seen": 1507328000
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023301257802156233,
+      "loss": 2.6021,
+      "theoretical_loss": 3.514286824545401,
+      "tokens_seen": 1508638720
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023277614904482693,
+      "loss": 2.5953,
+      "theoretical_loss": 3.5140180660825395,
+      "tokens_seen": 1509949440
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023253972006809156,
+      "loss": 2.5968,
+      "theoretical_loss": 3.5137496060744144,
+      "tokens_seen": 1511260160
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023230329109135618,
+      "loss": 2.6558,
+      "theoretical_loss": 3.5134814439311746,
+      "tokens_seen": 1512570880
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023206686211462078,
+      "loss": 2.6635,
+      "theoretical_loss": 3.513213579064643,
+      "tokens_seen": 1513881600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023183043313788538,
+      "loss": 2.6515,
+      "theoretical_loss": 3.512946010888313,
+      "tokens_seen": 1515192320
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023159400416115,
+      "loss": 2.6308,
+      "theoretical_loss": 3.51267873881734,
+      "tokens_seen": 1516503040
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002313575751844146,
+      "loss": 2.6905,
+      "theoretical_loss": 3.512411762268538,
+      "tokens_seen": 1517813760
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023112114620767922,
+      "loss": 2.6862,
+      "theoretical_loss": 3.512145080660371,
+      "tokens_seen": 1519124480
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023088471723094385,
+      "loss": 2.6154,
+      "theoretical_loss": 3.511878693412948,
+      "tokens_seen": 1520435200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023064828825420844,
+      "loss": 2.6091,
+      "theoretical_loss": 3.511612599948017,
+      "tokens_seen": 1521745920
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023041185927747304,
+      "loss": 2.6284,
+      "theoretical_loss": 3.5113467996889582,
+      "tokens_seen": 1523056640
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00023017543030073767,
+      "loss": 2.7055,
+      "theoretical_loss": 3.511081292060779,
+      "tokens_seen": 1524367360
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0002299390013240023,
+      "loss": 2.718,
+      "theoretical_loss": 3.5108160764901086,
+      "tokens_seen": 1525678080
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002297025723472669,
+      "loss": 2.695,
+      "theoretical_loss": 3.510551152405188,
+      "tokens_seen": 1526988800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002294661433705315,
+      "loss": 2.7101,
+      "theoretical_loss": 3.51028651923587,
+      "tokens_seen": 1528299520
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002292297143937961,
+      "loss": 2.6632,
+      "theoretical_loss": 3.5100221764136093,
+      "tokens_seen": 1529610240
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002289932854170607,
+      "loss": 2.723,
+      "theoretical_loss": 3.5097581233714576,
+      "tokens_seen": 1530920960
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002287568564403253,
+      "loss": 2.6842,
+      "theoretical_loss": 3.5094943595440578,
+      "tokens_seen": 1532231680
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022852042746358996,
+      "loss": 2.7223,
+      "theoretical_loss": 3.509230884367639,
+      "tokens_seen": 1533542400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022828399848685455,
+      "loss": 2.6696,
+      "theoretical_loss": 3.5089676972800086,
+      "tokens_seen": 1534853120
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022804756951011915,
+      "loss": 2.7184,
+      "theoretical_loss": 3.5087047977205486,
+      "tokens_seen": 1536163840
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022781114053338378,
+      "loss": 2.6775,
+      "theoretical_loss": 3.508442185130209,
+      "tokens_seen": 1537474560
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022757471155664837,
+      "loss": 2.7012,
+      "theoretical_loss": 3.508179858951503,
+      "tokens_seen": 1538785280
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.000227338282579913,
+      "loss": 2.6889,
+      "theoretical_loss": 3.5079178186284987,
+      "tokens_seen": 1540096000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022710185360317762,
+      "loss": 2.6993,
+      "theoretical_loss": 3.507656063606817,
+      "tokens_seen": 1541406720
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022686542462644222,
+      "loss": 2.6356,
+      "theoretical_loss": 3.5073945933336237,
+      "tokens_seen": 1542717440
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022662899564970682,
+      "loss": 2.6336,
+      "theoretical_loss": 3.5071334072576232,
+      "tokens_seen": 1544028160
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022639256667297144,
+      "loss": 2.7214,
+      "theoretical_loss": 3.5068725048290554,
+      "tokens_seen": 1545338880
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022615613769623604,
+      "loss": 2.6362,
+      "theoretical_loss": 3.506611885499689,
+      "tokens_seen": 1546649600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022591970871950066,
+      "loss": 2.604,
+      "theoretical_loss": 3.5063515487228143,
+      "tokens_seen": 1547960320
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002256832797427653,
+      "loss": 2.7294,
+      "theoretical_loss": 3.5060914939532406,
+      "tokens_seen": 1549271040
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002254468507660299,
+      "loss": 2.6265,
+      "theoretical_loss": 3.505831720647289,
+      "tokens_seen": 1550581760
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00022521042178929448,
+      "loss": 2.7019,
+      "theoretical_loss": 3.5055722282627864,
+      "tokens_seen": 1551892480
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0002249739928125591,
+      "loss": 2.6603,
+      "theoretical_loss": 3.5053130162590618,
+      "tokens_seen": 1553203200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022473756383582373,
+      "loss": 2.6934,
+      "theoretical_loss": 3.505054084096939,
+      "tokens_seen": 1554513920
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022450113485908833,
+      "loss": 2.6774,
+      "theoretical_loss": 3.504795431238733,
+      "tokens_seen": 1555824640
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022426470588235296,
+      "loss": 2.6292,
+      "theoretical_loss": 3.504537057148243,
+      "tokens_seen": 1557135360
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022402827690561755,
+      "loss": 2.6945,
+      "theoretical_loss": 3.504278961290749,
+      "tokens_seen": 1558446080
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022379184792888215,
+      "loss": 2.6436,
+      "theoretical_loss": 3.504021143133004,
+      "tokens_seen": 1559756800
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022355541895214677,
+      "loss": 2.6103,
+      "theoretical_loss": 3.5037636021432306,
+      "tokens_seen": 1561067520
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002233189899754114,
+      "loss": 2.6424,
+      "theoretical_loss": 3.5035063377911153,
+      "tokens_seen": 1562378240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.000223082560998676,
+      "loss": 2.6027,
+      "theoretical_loss": 3.5032493495478025,
+      "tokens_seen": 1563688960
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022284613202194062,
+      "loss": 2.6582,
+      "theoretical_loss": 3.50299263688589,
+      "tokens_seen": 1564999680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022260970304520522,
+      "loss": 2.6243,
+      "theoretical_loss": 3.5027361992794246,
+      "tokens_seen": 1566310400
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022237327406846982,
+      "loss": 2.6222,
+      "theoretical_loss": 3.5024800362038944,
+      "tokens_seen": 1567621120
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022213684509173447,
+      "loss": 2.6719,
+      "theoretical_loss": 3.5022241471362268,
+      "tokens_seen": 1568931840
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022190041611499907,
+      "loss": 2.5701,
+      "theoretical_loss": 3.5019685315547795,
+      "tokens_seen": 1570242560
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022166398713826366,
+      "loss": 2.6089,
+      "theoretical_loss": 3.5017131889393402,
+      "tokens_seen": 1571553280
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 868239,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.839005470275879,
+      "objective/train/theoretical_loss": 3.501458118771118,
+      "objective/train/tokens_used": 1593324000,
+      "theoretical_loss": 3.501458118771118,
+      "tokens_seen": 1572864000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002214275581615283,
+      "loss": 2.6126,
+      "theoretical_loss": 3.501458118771118,
+      "tokens_seen": 1572864000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022119112918479289,
+      "loss": 2.5811,
+      "theoretical_loss": 3.5012033205327375,
+      "tokens_seen": 1574174720
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.0002209547002080575,
+      "loss": 2.6021,
+      "theoretical_loss": 3.500948793708239,
+      "tokens_seen": 1575485440
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022071827123132213,
+      "loss": 2.5751,
+      "theoretical_loss": 3.5006945377830663,
+      "tokens_seen": 1576796160
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022048184225458673,
+      "loss": 2.5811,
+      "theoretical_loss": 3.5004405522440685,
+      "tokens_seen": 1578106880
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022024541327785133,
+      "loss": 2.5808,
+      "theoretical_loss": 3.5001868365794904,
+      "tokens_seen": 1579417600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00022000898430111595,
+      "loss": 2.6446,
+      "theoretical_loss": 3.499933390278968,
+      "tokens_seen": 1580728320
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021977255532438055,
+      "loss": 2.6177,
+      "theoretical_loss": 3.4996802128335274,
+      "tokens_seen": 1582039040
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021953612634764518,
+      "loss": 2.7024,
+      "theoretical_loss": 3.499427303735575,
+      "tokens_seen": 1583349760
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002192996973709098,
+      "loss": 2.6637,
+      "theoretical_loss": 3.499174662478895,
+      "tokens_seen": 1584660480
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002190632683941744,
+      "loss": 2.6836,
+      "theoretical_loss": 3.498922288558645,
+      "tokens_seen": 1585971200
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.000218826839417439,
+      "loss": 2.6956,
+      "theoretical_loss": 3.4986701814713492,
+      "tokens_seen": 1587281920
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021859041044070362,
+      "loss": 2.6254,
+      "theoretical_loss": 3.4984183407148954,
+      "tokens_seen": 1588592640
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021835398146396824,
+      "loss": 2.6858,
+      "theoretical_loss": 3.4981667657885307,
+      "tokens_seen": 1589903360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021811755248723284,
+      "loss": 2.6841,
+      "theoretical_loss": 3.497915456192853,
+      "tokens_seen": 1591214080
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021788112351049747,
+      "loss": 2.6944,
+      "theoretical_loss": 3.4976644114298114,
+      "tokens_seen": 1592524800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021764469453376206,
+      "loss": 2.6314,
+      "theoretical_loss": 3.497413631002696,
+      "tokens_seen": 1593835520
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021740826555702666,
+      "loss": 2.6018,
+      "theoretical_loss": 3.4971631144161393,
+      "tokens_seen": 1595146240
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021717183658029129,
+      "loss": 2.648,
+      "theoretical_loss": 3.4969128611761056,
+      "tokens_seen": 1596456960
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002169354076035559,
+      "loss": 2.6616,
+      "theoretical_loss": 3.4966628707898892,
+      "tokens_seen": 1597767680
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002166989786268205,
+      "loss": 2.6659,
+      "theoretical_loss": 3.4964131427661114,
+      "tokens_seen": 1599078400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002164625496500851,
+      "loss": 2.5814,
+      "theoretical_loss": 3.496163676614712,
+      "tokens_seen": 1600389120
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021622612067334973,
+      "loss": 2.6371,
+      "theoretical_loss": 3.4959144718469464,
+      "tokens_seen": 1601699840
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021598969169661433,
+      "loss": 2.5506,
+      "theoretical_loss": 3.4956655279753828,
+      "tokens_seen": 1603010560
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021575326271987895,
+      "loss": 2.6593,
+      "theoretical_loss": 3.4954168445138944,
+      "tokens_seen": 1604321280
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021551683374314358,
+      "loss": 2.613,
+      "theoretical_loss": 3.4951684209776577,
+      "tokens_seen": 1605632000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021528040476640817,
+      "loss": 2.5951,
+      "theoretical_loss": 3.494920256883145,
+      "tokens_seen": 1606942720
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00021504397578967277,
+      "loss": 2.5825,
+      "theoretical_loss": 3.4946723517481235,
+      "tokens_seen": 1608253440
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0002148075468129374,
+      "loss": 2.6113,
+      "theoretical_loss": 3.4944247050916477,
+      "tokens_seen": 1609564160
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.000214571117836202,
+      "loss": 2.6167,
+      "theoretical_loss": 3.4941773164340555,
+      "tokens_seen": 1610874880
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021433468885946662,
+      "loss": 2.69,
+      "theoretical_loss": 3.493930185296966,
+      "tokens_seen": 1612185600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021409825988273124,
+      "loss": 2.7004,
+      "theoretical_loss": 3.493683311203272,
+      "tokens_seen": 1613496320
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021386183090599584,
+      "loss": 2.6709,
+      "theoretical_loss": 3.493436693677137,
+      "tokens_seen": 1614807040
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021362540192926044,
+      "loss": 2.6444,
+      "theoretical_loss": 3.493190332243991,
+      "tokens_seen": 1616117760
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021338897295252506,
+      "loss": 2.6804,
+      "theoretical_loss": 3.492944226430525,
+      "tokens_seen": 1617428480
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002131525439757897,
+      "loss": 2.6555,
+      "theoretical_loss": 3.4926983757646894,
+      "tokens_seen": 1618739200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021291611499905428,
+      "loss": 2.649,
+      "theoretical_loss": 3.492452779775685,
+      "tokens_seen": 1620049920
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002126796860223189,
+      "loss": 2.6249,
+      "theoretical_loss": 3.492207437993964,
+      "tokens_seen": 1621360640
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002124432570455835,
+      "loss": 2.6787,
+      "theoretical_loss": 3.49196234995122,
+      "tokens_seen": 1622671360
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0002122068280688481,
+      "loss": 2.5892,
+      "theoretical_loss": 3.4917175151803903,
+      "tokens_seen": 1623982080
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021197039909211273,
+      "loss": 2.6202,
+      "theoretical_loss": 3.4914729332156442,
+      "tokens_seen": 1625292800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021173397011537735,
+      "loss": 2.635,
+      "theoretical_loss": 3.4912286035923863,
+      "tokens_seen": 1626603520
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021149754113864195,
+      "loss": 2.588,
+      "theoretical_loss": 3.490984525847245,
+      "tokens_seen": 1627914240
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021126111216190658,
+      "loss": 2.5813,
+      "theoretical_loss": 3.4907406995180743,
+      "tokens_seen": 1629224960
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021102468318517117,
+      "loss": 2.5951,
+      "theoretical_loss": 3.490497124143946,
+      "tokens_seen": 1630535680
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021078825420843577,
+      "loss": 2.6482,
+      "theoretical_loss": 3.4902537992651466,
+      "tokens_seen": 1631846400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021055182523170042,
+      "loss": 2.5608,
+      "theoretical_loss": 3.4900107244231737,
+      "tokens_seen": 1633157120
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021031539625496502,
+      "loss": 2.6163,
+      "theoretical_loss": 3.489767899160731,
+      "tokens_seen": 1634467840
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00021007896727822962,
+      "loss": 2.5978,
+      "theoretical_loss": 3.4895253230217236,
+      "tokens_seen": 1635778560
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00020984253830149424,
+      "loss": 2.6483,
+      "theoretical_loss": 3.489282995551256,
+      "tokens_seen": 1637089280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.45293069273957637,
+      "debugging/distinct-1-grams": 0.7681813940118882,
+      "debugging/distinct-2-grams": 0.9459392751721037,
+      "debugging/entropy-1-grams": 5.870395325680172,
+      "debugging/entropy-2-grams": 6.692635313430856,
+      "debugging/length": 498.1666666666667,
+      "debugging/num_segments": 12,
+      "epoch": 0.59,
+      "objective/train/docs_used": 903766,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7166571617126465,
+      "objective/train/theoretical_loss": 3.4890409162956253,
+      "objective/train/tokens_used": 1658860000,
+      "theoretical_loss": 3.4890409162956253,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020960610932475884,
+      "loss": 2.6502,
+      "theoretical_loss": 3.4890409162956253,
+      "tokens_seen": 1638400000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020936968034802346,
+      "loss": 2.5996,
+      "theoretical_loss": 3.48879908480232,
+      "tokens_seen": 1639710720
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002091332513712881,
+      "loss": 2.5989,
+      "theoretical_loss": 3.4885575006200136,
+      "tokens_seen": 1641021440
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020889682239455269,
+      "loss": 2.6446,
+      "theoretical_loss": 3.488316163298561,
+      "tokens_seen": 1642332160
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020866039341781728,
+      "loss": 2.6045,
+      "theoretical_loss": 3.488075072388996,
+      "tokens_seen": 1643642880
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002084239644410819,
+      "loss": 2.6317,
+      "theoretical_loss": 3.487834227443525,
+      "tokens_seen": 1644953600
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002081875354643465,
+      "loss": 2.5895,
+      "theoretical_loss": 3.4875936280155244,
+      "tokens_seen": 1646264320
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020795110648761113,
+      "loss": 2.6314,
+      "theoretical_loss": 3.4873532736595374,
+      "tokens_seen": 1647575040
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020771467751087575,
+      "loss": 2.6207,
+      "theoretical_loss": 3.4871131639312676,
+      "tokens_seen": 1648885760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020747824853414035,
+      "loss": 2.5984,
+      "theoretical_loss": 3.486873298387577,
+      "tokens_seen": 1650196480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020724181955740495,
+      "loss": 2.6314,
+      "theoretical_loss": 3.4866336765864814,
+      "tokens_seen": 1651507200
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020700539058066957,
+      "loss": 2.6012,
+      "theoretical_loss": 3.4863942980871463,
+      "tokens_seen": 1652817920
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002067689616039342,
+      "loss": 2.6547,
+      "theoretical_loss": 3.4861551624498843,
+      "tokens_seen": 1654128640
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0002065325326271988,
+      "loss": 2.6315,
+      "theoretical_loss": 3.485916269236149,
+      "tokens_seen": 1655439360
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020629610365046342,
+      "loss": 2.6762,
+      "theoretical_loss": 3.485677618008533,
+      "tokens_seen": 1656750080
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020605967467372802,
+      "loss": 2.6716,
+      "theoretical_loss": 3.4854392083307637,
+      "tokens_seen": 1658060800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020582324569699262,
+      "loss": 2.6174,
+      "theoretical_loss": 3.4852010397676976,
+      "tokens_seen": 1659371520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020558681672025724,
+      "loss": 2.5947,
+      "theoretical_loss": 3.4849631118853193,
+      "tokens_seen": 1660682240
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020535038774352186,
+      "loss": 2.652,
+      "theoretical_loss": 3.484725424250737,
+      "tokens_seen": 1661992960
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020511395876678646,
+      "loss": 2.6665,
+      "theoretical_loss": 3.4844879764321766,
+      "tokens_seen": 1663303680
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020487752979005106,
+      "loss": 2.6389,
+      "theoretical_loss": 3.4842507679989803,
+      "tokens_seen": 1664614400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00020464110081331568,
+      "loss": 2.6247,
+      "theoretical_loss": 3.4840137985216013,
+      "tokens_seen": 1665925120
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020440467183658028,
+      "loss": 2.5557,
+      "theoretical_loss": 3.4837770675716015,
+      "tokens_seen": 1667235840
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002041682428598449,
+      "loss": 2.6202,
+      "theoretical_loss": 3.4835405747216477,
+      "tokens_seen": 1668546560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020393181388310953,
+      "loss": 2.642,
+      "theoretical_loss": 3.483304319545505,
+      "tokens_seen": 1669857280
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020369538490637413,
+      "loss": 2.6105,
+      "theoretical_loss": 3.4830683016180375,
+      "tokens_seen": 1671168000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020345895592963873,
+      "loss": 2.5464,
+      "theoretical_loss": 3.4828325205152013,
+      "tokens_seen": 1672478720
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020322252695290335,
+      "loss": 2.5557,
+      "theoretical_loss": 3.482596975814042,
+      "tokens_seen": 1673789440
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020298609797616795,
+      "loss": 2.5785,
+      "theoretical_loss": 3.4823616670926913,
+      "tokens_seen": 1675100160
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020274966899943257,
+      "loss": 2.5894,
+      "theoretical_loss": 3.4821265939303636,
+      "tokens_seen": 1676410880
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002025132400226972,
+      "loss": 2.6049,
+      "theoretical_loss": 3.48189175590735,
+      "tokens_seen": 1677721600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002022768110459618,
+      "loss": 2.6027,
+      "theoretical_loss": 3.481657152605019,
+      "tokens_seen": 1679032320
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002020403820692264,
+      "loss": 2.561,
+      "theoretical_loss": 3.481422783605809,
+      "tokens_seen": 1680343040
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020180395309249102,
+      "loss": 2.6064,
+      "theoretical_loss": 3.4811886484932266,
+      "tokens_seen": 1681653760
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020156752411575564,
+      "loss": 2.5923,
+      "theoretical_loss": 3.4809547468518423,
+      "tokens_seen": 1682964480
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020133109513902024,
+      "loss": 2.6181,
+      "theoretical_loss": 3.480721078267288,
+      "tokens_seen": 1684275200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020109466616228486,
+      "loss": 2.5912,
+      "theoretical_loss": 3.4804876423262523,
+      "tokens_seen": 1685585920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020085823718554946,
+      "loss": 2.5924,
+      "theoretical_loss": 3.480254438616477,
+      "tokens_seen": 1686896640
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020062180820881406,
+      "loss": 2.6505,
+      "theoretical_loss": 3.480021466726755,
+      "tokens_seen": 1688207360
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00020038537923207868,
+      "loss": 2.6247,
+      "theoretical_loss": 3.479788726246925,
+      "tokens_seen": 1689518080
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0002001489502553433,
+      "loss": 2.6109,
+      "theoretical_loss": 3.479556216767869,
+      "tokens_seen": 1690828800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001999125212786079,
+      "loss": 2.6633,
+      "theoretical_loss": 3.47932393788151,
+      "tokens_seen": 1692139520
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00019967609230187253,
+      "loss": 2.6257,
+      "theoretical_loss": 3.4790918891808054,
+      "tokens_seen": 1693450240
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019943966332513713,
+      "loss": 2.6146,
+      "theoretical_loss": 3.4788600702597465,
+      "tokens_seen": 1694760960
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019920323434840172,
+      "loss": 2.6634,
+      "theoretical_loss": 3.4786284807133536,
+      "tokens_seen": 1696071680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019896680537166638,
+      "loss": 2.6493,
+      "theoretical_loss": 3.4783971201376738,
+      "tokens_seen": 1697382400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019873037639493097,
+      "loss": 2.6267,
+      "theoretical_loss": 3.478165988129776,
+      "tokens_seen": 1698693120
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019849394741819557,
+      "loss": 2.5734,
+      "theoretical_loss": 3.477935084287748,
+      "tokens_seen": 1700003840
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001982575184414602,
+      "loss": 2.5749,
+      "theoretical_loss": 3.4777044082106956,
+      "tokens_seen": 1701314560
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001980210894647248,
+      "loss": 2.5672,
+      "theoretical_loss": 3.4774739594987345,
+      "tokens_seen": 1702625280
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 939852,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.528791904449463,
+      "objective/train/theoretical_loss": 3.477243737752991,
+      "objective/train/tokens_used": 1724396000,
+      "theoretical_loss": 3.477243737752991,
+      "tokens_seen": 1703936000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019778466048798942,
+      "loss": 2.5337,
+      "theoretical_loss": 3.477243737752991,
+      "tokens_seen": 1703936000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019754823151125404,
+      "loss": 2.5966,
+      "theoretical_loss": 3.477013742575597,
+      "tokens_seen": 1705246720
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019731180253451864,
+      "loss": 2.5832,
+      "theoretical_loss": 3.4767839735696877,
+      "tokens_seen": 1706557440
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019707537355778324,
+      "loss": 2.5544,
+      "theoretical_loss": 3.4765544303393963,
+      "tokens_seen": 1707868160
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019683894458104786,
+      "loss": 2.5838,
+      "theoretical_loss": 3.476325112489853,
+      "tokens_seen": 1709178880
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019660251560431246,
+      "loss": 2.629,
+      "theoretical_loss": 3.476096019627181,
+      "tokens_seen": 1710489600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019636608662757708,
+      "loss": 2.5708,
+      "theoretical_loss": 3.475867151358491,
+      "tokens_seen": 1711800320
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001961296576508417,
+      "loss": 2.5992,
+      "theoretical_loss": 3.475638507291883,
+      "tokens_seen": 1713111040
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001958932286741063,
+      "loss": 2.601,
+      "theoretical_loss": 3.4754100870364373,
+      "tokens_seen": 1714421760
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.0001956567996973709,
+      "loss": 2.5865,
+      "theoretical_loss": 3.475181890202215,
+      "tokens_seen": 1715732480
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019542037072063553,
+      "loss": 2.5905,
+      "theoretical_loss": 3.474953916400255,
+      "tokens_seen": 1717043200
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019518394174390015,
+      "loss": 2.6194,
+      "theoretical_loss": 3.4747261652425676,
+      "tokens_seen": 1718353920
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019494751276716475,
+      "loss": 2.6061,
+      "theoretical_loss": 3.474498636342134,
+      "tokens_seen": 1719664640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00019471108379042937,
+      "loss": 2.569,
+      "theoretical_loss": 3.474271329312904,
+      "tokens_seen": 1720975360
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019447465481369397,
+      "loss": 2.6244,
+      "theoretical_loss": 3.4740442437697894,
+      "tokens_seen": 1722286080
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019423822583695857,
+      "loss": 2.6409,
+      "theoretical_loss": 3.4738173793286626,
+      "tokens_seen": 1723596800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001940017968602232,
+      "loss": 2.6267,
+      "theoretical_loss": 3.4735907356063556,
+      "tokens_seen": 1724907520
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019376536788348782,
+      "loss": 2.633,
+      "theoretical_loss": 3.4733643122206534,
+      "tokens_seen": 1726218240
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019352893890675242,
+      "loss": 2.5952,
+      "theoretical_loss": 3.473138108790293,
+      "tokens_seen": 1727528960
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.000193292509930017,
+      "loss": 2.5959,
+      "theoretical_loss": 3.4729121249349597,
+      "tokens_seen": 1728839680
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019305608095328164,
+      "loss": 2.5711,
+      "theoretical_loss": 3.4726863602752838,
+      "tokens_seen": 1730150400
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019281965197654624,
+      "loss": 2.681,
+      "theoretical_loss": 3.4724608144328384,
+      "tokens_seen": 1731461120
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019258322299981086,
+      "loss": 2.6863,
+      "theoretical_loss": 3.4722354870301353,
+      "tokens_seen": 1732771840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019234679402307548,
+      "loss": 2.6406,
+      "theoretical_loss": 3.4720103776906224,
+      "tokens_seen": 1734082560
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019211036504634008,
+      "loss": 2.6842,
+      "theoretical_loss": 3.471785486038681,
+      "tokens_seen": 1735393280
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019187393606960468,
+      "loss": 2.5764,
+      "theoretical_loss": 3.4715608116996224,
+      "tokens_seen": 1736704000
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001916375070928693,
+      "loss": 2.6103,
+      "theoretical_loss": 3.4713363542996847,
+      "tokens_seen": 1738014720
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001914010781161339,
+      "loss": 2.6482,
+      "theoretical_loss": 3.47111211346603,
+      "tokens_seen": 1739325440
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019116464913939853,
+      "loss": 2.6439,
+      "theoretical_loss": 3.470888088826743,
+      "tokens_seen": 1740636160
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019092822016266315,
+      "loss": 2.5951,
+      "theoretical_loss": 3.4706642800108245,
+      "tokens_seen": 1741946880
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019069179118592775,
+      "loss": 2.571,
+      "theoretical_loss": 3.4704406866481916,
+      "tokens_seen": 1743257600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019045536220919235,
+      "loss": 2.5665,
+      "theoretical_loss": 3.470217308369673,
+      "tokens_seen": 1744568320
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00019021893323245697,
+      "loss": 2.6195,
+      "theoretical_loss": 3.469994144807007,
+      "tokens_seen": 1745879040
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001899825042557216,
+      "loss": 2.6537,
+      "theoretical_loss": 3.46977119559284,
+      "tokens_seen": 1747189760
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001897460752789862,
+      "loss": 2.6125,
+      "theoretical_loss": 3.4695484603607185,
+      "tokens_seen": 1748500480
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00018950964630225082,
+      "loss": 2.6301,
+      "theoretical_loss": 3.4693259387450923,
+      "tokens_seen": 1749811200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018927321732551541,
+      "loss": 2.6411,
+      "theoretical_loss": 3.4691036303813076,
+      "tokens_seen": 1751121920
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018903678834878,
+      "loss": 2.6629,
+      "theoretical_loss": 3.4688815349056066,
+      "tokens_seen": 1752432640
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018880035937204464,
+      "loss": 2.583,
+      "theoretical_loss": 3.4686596519551216,
+      "tokens_seen": 1753743360
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018856393039530926,
+      "loss": 2.5861,
+      "theoretical_loss": 3.4684379811678756,
+      "tokens_seen": 1755054080
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018832750141857386,
+      "loss": 2.6385,
+      "theoretical_loss": 3.468216522182778,
+      "tokens_seen": 1756364800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018809107244183848,
+      "loss": 2.6729,
+      "theoretical_loss": 3.4679952746396197,
+      "tokens_seen": 1757675520
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018785464346510308,
+      "loss": 2.6283,
+      "theoretical_loss": 3.4677742381790746,
+      "tokens_seen": 1758986240
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018761821448836768,
+      "loss": 2.6461,
+      "theoretical_loss": 3.467553412442693,
+      "tokens_seen": 1760296960
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018738178551163233,
+      "loss": 2.5962,
+      "theoretical_loss": 3.467332797072901,
+      "tokens_seen": 1761607680
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018714535653489693,
+      "loss": 2.6243,
+      "theoretical_loss": 3.4671123917129956,
+      "tokens_seen": 1762918400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018690892755816152,
+      "loss": 2.5963,
+      "theoretical_loss": 3.4668921960071453,
+      "tokens_seen": 1764229120
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018667249858142615,
+      "loss": 2.6143,
+      "theoretical_loss": 3.4666722096003837,
+      "tokens_seen": 1765539840
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018643606960469075,
+      "loss": 2.6132,
+      "theoretical_loss": 3.4664524321386088,
+      "tokens_seen": 1766850560
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018619964062795537,
+      "loss": 2.5981,
+      "theoretical_loss": 3.4662328632685795,
+      "tokens_seen": 1768161280
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 974565,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.4235711097717285,
+      "objective/train/theoretical_loss": 3.4660135026379137,
+      "objective/train/tokens_used": 1789932000,
+      "theoretical_loss": 3.4660135026379137,
+      "tokens_seen": 1769472000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018596321165122,
+      "loss": 2.6311,
+      "theoretical_loss": 3.4660135026379137,
+      "tokens_seen": 1769472000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001857267826744846,
+      "loss": 2.6191,
+      "theoretical_loss": 3.4657943498950856,
+      "tokens_seen": 1770782720
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001854903536977492,
+      "loss": 2.6494,
+      "theoretical_loss": 3.465575404689421,
+      "tokens_seen": 1772093440
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018525392472101382,
+      "loss": 2.6746,
+      "theoretical_loss": 3.465356666671097,
+      "tokens_seen": 1773404160
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.0001850174957442784,
+      "loss": 2.6448,
+      "theoretical_loss": 3.465138135491137,
+      "tokens_seen": 1774714880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018478106676754304,
+      "loss": 2.6297,
+      "theoretical_loss": 3.4649198108014123,
+      "tokens_seen": 1776025600
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00018454463779080766,
+      "loss": 2.6101,
+      "theoretical_loss": 3.464701692254634,
+      "tokens_seen": 1777336320
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018430820881407226,
+      "loss": 2.6554,
+      "theoretical_loss": 3.464483779504353,
+      "tokens_seen": 1778647040
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018407177983733686,
+      "loss": 2.6626,
+      "theoretical_loss": 3.4642660722049583,
+      "tokens_seen": 1779957760
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018383535086060148,
+      "loss": 2.6849,
+      "theoretical_loss": 3.4640485700116725,
+      "tokens_seen": 1781268480
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001835989218838661,
+      "loss": 2.6523,
+      "theoretical_loss": 3.46383127258055,
+      "tokens_seen": 1782579200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001833624929071307,
+      "loss": 2.6269,
+      "theoretical_loss": 3.4636141795684754,
+      "tokens_seen": 1783889920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018312606393039533,
+      "loss": 2.6267,
+      "theoretical_loss": 3.463397290633157,
+      "tokens_seen": 1785200640
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018288963495365993,
+      "loss": 2.6707,
+      "theoretical_loss": 3.4631806054331316,
+      "tokens_seen": 1786511360
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018265320597692452,
+      "loss": 2.5844,
+      "theoretical_loss": 3.4629641236277524,
+      "tokens_seen": 1787822080
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018241677700018915,
+      "loss": 2.6576,
+      "theoretical_loss": 3.4627478448771947,
+      "tokens_seen": 1789132800
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018218034802345377,
+      "loss": 2.6245,
+      "theoretical_loss": 3.4625317688424486,
+      "tokens_seen": 1790443520
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018194391904671837,
+      "loss": 2.6364,
+      "theoretical_loss": 3.462315895185318,
+      "tokens_seen": 1791754240
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.000181707490069983,
+      "loss": 2.671,
+      "theoretical_loss": 3.4621002235684184,
+      "tokens_seen": 1793064960
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001814710610932476,
+      "loss": 2.6663,
+      "theoretical_loss": 3.4618847536551733,
+      "tokens_seen": 1794375680
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001812346321165122,
+      "loss": 2.7,
+      "theoretical_loss": 3.4616694851098124,
+      "tokens_seen": 1795686400
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018099820313977681,
+      "loss": 2.6609,
+      "theoretical_loss": 3.4614544175973694,
+      "tokens_seen": 1796997120
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018076177416304144,
+      "loss": 2.6348,
+      "theoretical_loss": 3.461239550783678,
+      "tokens_seen": 1798307840
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018052534518630604,
+      "loss": 2.6882,
+      "theoretical_loss": 3.4610248843353713,
+      "tokens_seen": 1799618560
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018028891620957063,
+      "loss": 2.6301,
+      "theoretical_loss": 3.460810417919878,
+      "tokens_seen": 1800929280
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00018006430868167204,
+      "loss": 2.6542,
+      "theoretical_loss": 3.4605961512054204,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017982787970493663,
+      "loss": 2.6641,
+      "theoretical_loss": 3.4603820838610115,
+      "tokens_seen": 1803550720
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00017959145072820123,
+      "loss": 2.6625,
+      "theoretical_loss": 3.460168215556455,
+      "tokens_seen": 1804861440
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017935502175146586,
+      "loss": 2.6764,
+      "theoretical_loss": 3.4599545459623364,
+      "tokens_seen": 1806172160
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017911859277473048,
+      "loss": 2.6512,
+      "theoretical_loss": 3.4597410747500295,
+      "tokens_seen": 1807482880
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017888216379799508,
+      "loss": 2.6818,
+      "theoretical_loss": 3.459527801591687,
+      "tokens_seen": 1808793600
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001786457348212597,
+      "loss": 2.6151,
+      "theoretical_loss": 3.4593147261602404,
+      "tokens_seen": 1810104320
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001784093058445243,
+      "loss": 2.5853,
+      "theoretical_loss": 3.459101848129399,
+      "tokens_seen": 1811415040
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001781728768677889,
+      "loss": 2.6714,
+      "theoretical_loss": 3.4588891671736457,
+      "tokens_seen": 1812725760
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017793644789105355,
+      "loss": 2.6643,
+      "theoretical_loss": 3.4586766829682336,
+      "tokens_seen": 1814036480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017770001891431815,
+      "loss": 2.6514,
+      "theoretical_loss": 3.4584643951891865,
+      "tokens_seen": 1815347200
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017746358993758274,
+      "loss": 2.5753,
+      "theoretical_loss": 3.4582523035132953,
+      "tokens_seen": 1816657920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017722716096084737,
+      "loss": 2.6513,
+      "theoretical_loss": 3.458040407618115,
+      "tokens_seen": 1817968640
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017699073198411197,
+      "loss": 2.6728,
+      "theoretical_loss": 3.457828707181962,
+      "tokens_seen": 1819279360
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017675430300737656,
+      "loss": 2.5972,
+      "theoretical_loss": 3.4576172018839135,
+      "tokens_seen": 1820590080
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017651787403064122,
+      "loss": 2.6562,
+      "theoretical_loss": 3.4574058914038037,
+      "tokens_seen": 1821900800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001762814450539058,
+      "loss": 2.6697,
+      "theoretical_loss": 3.4571947754222228,
+      "tokens_seen": 1823211520
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001760450160771704,
+      "loss": 2.5893,
+      "theoretical_loss": 3.456983853620513,
+      "tokens_seen": 1824522240
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017580858710043503,
+      "loss": 2.6081,
+      "theoretical_loss": 3.4567731256807663,
+      "tokens_seen": 1825832960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017557215812369963,
+      "loss": 2.5902,
+      "theoretical_loss": 3.4565625912858255,
+      "tokens_seen": 1827143680
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017533572914696426,
+      "loss": 2.6056,
+      "theoretical_loss": 3.4563522501192754,
+      "tokens_seen": 1828454400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017509930017022888,
+      "loss": 2.5993,
+      "theoretical_loss": 3.456142101865449,
+      "tokens_seen": 1829765120
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017486287119349348,
+      "loss": 2.5805,
+      "theoretical_loss": 3.4559321462094164,
+      "tokens_seen": 1831075840
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00017462644221675808,
+      "loss": 2.6069,
+      "theoretical_loss": 3.4557223828369894,
+      "tokens_seen": 1832386560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001743900132400227,
+      "loss": 2.5635,
+      "theoretical_loss": 3.4555128114347164,
+      "tokens_seen": 1833697280
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 1010128,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5836546421051025,
+      "objective/train/theoretical_loss": 3.455303431689879,
+      "objective/train/tokens_used": 1855468000,
+      "theoretical_loss": 3.455303431689879,
+      "tokens_seen": 1835008000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017415358426328733,
+      "loss": 2.5874,
+      "theoretical_loss": 3.455303431689879,
+      "tokens_seen": 1835008000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017391715528655192,
+      "loss": 2.6238,
+      "theoretical_loss": 3.4550942432904925,
+      "tokens_seen": 1836318720
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017368072630981655,
+      "loss": 2.6155,
+      "theoretical_loss": 3.4548852459253014,
+      "tokens_seen": 1837629440
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017344429733308115,
+      "loss": 2.6122,
+      "theoretical_loss": 3.4546764392837783,
+      "tokens_seen": 1838940160
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017320786835634574,
+      "loss": 2.6004,
+      "theoretical_loss": 3.454467823056122,
+      "tokens_seen": 1840250880
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017297143937961037,
+      "loss": 2.5447,
+      "theoretical_loss": 3.4542593969332542,
+      "tokens_seen": 1841561600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.000172735010402875,
+      "loss": 2.5652,
+      "theoretical_loss": 3.4540511606068174,
+      "tokens_seen": 1842872320
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001724985814261396,
+      "loss": 2.6094,
+      "theoretical_loss": 3.4538431137691745,
+      "tokens_seen": 1844183040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017226215244940421,
+      "loss": 2.5951,
+      "theoretical_loss": 3.4536352561134036,
+      "tokens_seen": 1845493760
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001720257234726688,
+      "loss": 2.5887,
+      "theoretical_loss": 3.453427587333298,
+      "tokens_seen": 1846804480
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001717892944959334,
+      "loss": 2.5725,
+      "theoretical_loss": 3.453220107123365,
+      "tokens_seen": 1848115200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017155286551919806,
+      "loss": 2.6188,
+      "theoretical_loss": 3.4530128151788193,
+      "tokens_seen": 1849425920
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017131643654246266,
+      "loss": 2.5856,
+      "theoretical_loss": 3.452805711195586,
+      "tokens_seen": 1850736640
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017108000756572726,
+      "loss": 2.5942,
+      "theoretical_loss": 3.4525987948702963,
+      "tokens_seen": 1852047360
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017084357858899188,
+      "loss": 2.6117,
+      "theoretical_loss": 3.452392065900284,
+      "tokens_seen": 1853358080
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017060714961225648,
+      "loss": 2.5475,
+      "theoretical_loss": 3.4521855239835855,
+      "tokens_seen": 1854668800
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017037072063552107,
+      "loss": 2.5778,
+      "theoretical_loss": 3.4519791688189363,
+      "tokens_seen": 1855979520
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00017013429165878573,
+      "loss": 2.5803,
+      "theoretical_loss": 3.45177300010577,
+      "tokens_seen": 1857290240
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016989786268205032,
+      "loss": 2.5299,
+      "theoretical_loss": 3.4515670175442157,
+      "tokens_seen": 1858600960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001696732551541517,
+      "loss": 2.5795,
+      "theoretical_loss": 3.4513612208350946,
+      "tokens_seen": 1859911680
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00016943682617741632,
+      "loss": 2.5617,
+      "theoretical_loss": 3.4511556096799216,
+      "tokens_seen": 1861222400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016920039720068092,
+      "loss": 2.5692,
+      "theoretical_loss": 3.450950183780898,
+      "tokens_seen": 1862533120
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016896396822394552,
+      "loss": 2.5672,
+      "theoretical_loss": 3.450744942840913,
+      "tokens_seen": 1863843840
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016872753924721014,
+      "loss": 2.5834,
+      "theoretical_loss": 3.450539886563542,
+      "tokens_seen": 1865154560
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016849111027047477,
+      "loss": 2.5957,
+      "theoretical_loss": 3.450335014653043,
+      "tokens_seen": 1866465280
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016826650274257612,
+      "loss": 2.5618,
+      "theoretical_loss": 3.4501303268143535,
+      "tokens_seen": 1867776000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016803007376584077,
+      "loss": 2.6263,
+      "theoretical_loss": 3.449925822753091,
+      "tokens_seen": 1869086720
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016779364478910537,
+      "loss": 2.5361,
+      "theoretical_loss": 3.4497215021755494,
+      "tokens_seen": 1870397440
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016755721581236996,
+      "loss": 2.6208,
+      "theoretical_loss": 3.4495173647886985,
+      "tokens_seen": 1871708160
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001673207868356346,
+      "loss": 2.5855,
+      "theoretical_loss": 3.449313410300179,
+      "tokens_seen": 1873018880
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016708435785889919,
+      "loss": 2.5791,
+      "theoretical_loss": 3.449109638418304,
+      "tokens_seen": 1874329600
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016684792888216378,
+      "loss": 2.5693,
+      "theoretical_loss": 3.448906048852054,
+      "tokens_seen": 1875640320
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016661149990542843,
+      "loss": 2.5298,
+      "theoretical_loss": 3.448702641311077,
+      "tokens_seen": 1876951040
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016637507092869303,
+      "loss": 2.6534,
+      "theoretical_loss": 3.4484994155056867,
+      "tokens_seen": 1878261760
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016613864195195763,
+      "loss": 2.6041,
+      "theoretical_loss": 3.4482963711468573,
+      "tokens_seen": 1879572480
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016590221297522223,
+      "loss": 2.5779,
+      "theoretical_loss": 3.4480935079462256,
+      "tokens_seen": 1880883200
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016566578399848685,
+      "loss": 2.6065,
+      "theoretical_loss": 3.4478908256160867,
+      "tokens_seen": 1882193920
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016542935502175148,
+      "loss": 2.6268,
+      "theoretical_loss": 3.447688323869392,
+      "tokens_seen": 1883504640
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00016519292604501607,
+      "loss": 2.6228,
+      "theoretical_loss": 3.4474860024197485,
+      "tokens_seen": 1884815360
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001649564970682807,
+      "loss": 2.5862,
+      "theoretical_loss": 3.4472838609814165,
+      "tokens_seen": 1886126080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001647200680915453,
+      "loss": 2.6075,
+      "theoretical_loss": 3.447081899269307,
+      "tokens_seen": 1887436800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001644836391148099,
+      "loss": 2.6235,
+      "theoretical_loss": 3.4468801169989782,
+      "tokens_seen": 1888747520
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016424721013807455,
+      "loss": 2.5954,
+      "theoretical_loss": 3.446678513886639,
+      "tokens_seen": 1890058240
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016401078116133914,
+      "loss": 2.5969,
+      "theoretical_loss": 3.4464770896491403,
+      "tokens_seen": 1891368960
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016377435218460374,
+      "loss": 2.6059,
+      "theoretical_loss": 3.4462758440039787,
+      "tokens_seen": 1892679680
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016353792320786836,
+      "loss": 2.6101,
+      "theoretical_loss": 3.446074776669291,
+      "tokens_seen": 1893990400
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016330149423113296,
+      "loss": 2.6146,
+      "theoretical_loss": 3.4458738873638532,
+      "tokens_seen": 1895301120
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016306506525439756,
+      "loss": 2.6096,
+      "theoretical_loss": 3.4456731758070793,
+      "tokens_seen": 1896611840
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001628286362776622,
+      "loss": 2.6357,
+      "theoretical_loss": 3.4454726417190202,
+      "tokens_seen": 1897922560
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001625922073009268,
+      "loss": 2.6779,
+      "theoretical_loss": 3.445272284820359,
+      "tokens_seen": 1899233280
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 1045665,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.5012683868408203,
+      "objective/train/theoretical_loss": 3.4450721048324118,
+      "objective/train/tokens_used": 1921004000,
+      "theoretical_loss": 3.4450721048324118,
+      "tokens_seen": 1900544000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001623557783241914,
+      "loss": 2.6911,
+      "theoretical_loss": 3.4450721048324118,
+      "tokens_seen": 1900544000
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016211934934745603,
+      "loss": 2.6174,
+      "theoretical_loss": 3.4448721014771233,
+      "tokens_seen": 1901854720
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016188292037072063,
+      "loss": 2.6818,
+      "theoretical_loss": 3.4446722744770693,
+      "tokens_seen": 1903165440
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016164649139398525,
+      "loss": 2.6656,
+      "theoretical_loss": 3.444472623555449,
+      "tokens_seen": 1904476160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016141006241724988,
+      "loss": 2.6631,
+      "theoretical_loss": 3.444273148436088,
+      "tokens_seen": 1905786880
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016117363344051447,
+      "loss": 2.6765,
+      "theoretical_loss": 3.444073848843434,
+      "tokens_seen": 1907097600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016093720446377907,
+      "loss": 2.725,
+      "theoretical_loss": 3.4438747245025563,
+      "tokens_seen": 1908408320
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001607007754870437,
+      "loss": 2.6956,
+      "theoretical_loss": 3.443675775139142,
+      "tokens_seen": 1909719040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001604643465103083,
+      "loss": 2.6756,
+      "theoretical_loss": 3.443477000479497,
+      "tokens_seen": 1911029760
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00016022791753357292,
+      "loss": 2.6232,
+      "theoretical_loss": 3.443278400250541,
+      "tokens_seen": 1912340480
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015999148855683754,
+      "loss": 2.6492,
+      "theoretical_loss": 3.443079974179809,
+      "tokens_seen": 1913651200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015975505958010214,
+      "loss": 2.6126,
+      "theoretical_loss": 3.4428817219954455,
+      "tokens_seen": 1914961920
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015951863060336674,
+      "loss": 2.6542,
+      "theoretical_loss": 3.4426836434262085,
+      "tokens_seen": 1916272640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00015928220162663136,
+      "loss": 2.6724,
+      "theoretical_loss": 3.4424857382014613,
+      "tokens_seen": 1917583360
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.000159045772649896,
+      "loss": 2.6582,
+      "theoretical_loss": 3.442288006051175,
+      "tokens_seen": 1918894080
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015880934367316059,
+      "loss": 2.5823,
+      "theoretical_loss": 3.442090446705925,
+      "tokens_seen": 1920204800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001585729146964252,
+      "loss": 2.5698,
+      "theoretical_loss": 3.44189305989689,
+      "tokens_seen": 1921515520
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001583364857196898,
+      "loss": 2.5542,
+      "theoretical_loss": 3.4416958453558495,
+      "tokens_seen": 1922826240
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001581000567429544,
+      "loss": 2.624,
+      "theoretical_loss": 3.441498802815183,
+      "tokens_seen": 1924136960
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015786362776621903,
+      "loss": 2.5769,
+      "theoretical_loss": 3.4413019320078666,
+      "tokens_seen": 1925447680
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015762719878948365,
+      "loss": 2.5947,
+      "theoretical_loss": 3.441105232667474,
+      "tokens_seen": 1926758400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015739076981274825,
+      "loss": 2.5473,
+      "theoretical_loss": 3.440908704528172,
+      "tokens_seen": 1928069120
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015715434083601288,
+      "loss": 2.5087,
+      "theoretical_loss": 3.4407123473247196,
+      "tokens_seen": 1929379840
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015691791185927747,
+      "loss": 2.5753,
+      "theoretical_loss": 3.440516160792468,
+      "tokens_seen": 1930690560
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015668148288254207,
+      "loss": 2.54,
+      "theoretical_loss": 3.4403201446673553,
+      "tokens_seen": 1932001280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015644505390580672,
+      "loss": 2.5432,
+      "theoretical_loss": 3.440124298685909,
+      "tokens_seen": 1933312000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015620862492907132,
+      "loss": 2.5685,
+      "theoretical_loss": 3.4399286225852412,
+      "tokens_seen": 1934622720
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015597219595233592,
+      "loss": 2.512,
+      "theoretical_loss": 3.4397331161030484,
+      "tokens_seen": 1935933440
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015573576697560054,
+      "loss": 2.5705,
+      "theoretical_loss": 3.4395377789776083,
+      "tokens_seen": 1937244160
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015549933799886514,
+      "loss": 2.5155,
+      "theoretical_loss": 3.439342610947781,
+      "tokens_seen": 1938554880
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015526290902212974,
+      "loss": 2.5818,
+      "theoretical_loss": 3.439147611753004,
+      "tokens_seen": 1939865600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001550264800453944,
+      "loss": 2.6435,
+      "theoretical_loss": 3.4389527811332923,
+      "tokens_seen": 1941176320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015479005106865899,
+      "loss": 2.5728,
+      "theoretical_loss": 3.4387581188292367,
+      "tokens_seen": 1942487040
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015455362209192358,
+      "loss": 2.5587,
+      "theoretical_loss": 3.438563624582002,
+      "tokens_seen": 1943797760
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00015431719311518818,
+      "loss": 2.5838,
+      "theoretical_loss": 3.438369298133325,
+      "tokens_seen": 1945108480
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001540807641384528,
+      "loss": 2.634,
+      "theoretical_loss": 3.438175139225512,
+      "tokens_seen": 1946419200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015384433516171743,
+      "loss": 2.5746,
+      "theoretical_loss": 3.4379811476014415,
+      "tokens_seen": 1947729920
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015360790618498203,
+      "loss": 2.6395,
+      "theoretical_loss": 3.4377873230045553,
+      "tokens_seen": 1949040640
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015337147720824665,
+      "loss": 2.5613,
+      "theoretical_loss": 3.4375936651788637,
+      "tokens_seen": 1950351360
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015313504823151125,
+      "loss": 2.6669,
+      "theoretical_loss": 3.437400173868939,
+      "tokens_seen": 1951662080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015289861925477585,
+      "loss": 2.5738,
+      "theoretical_loss": 3.437206848819918,
+      "tokens_seen": 1952972800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001526621902780405,
+      "loss": 2.5354,
+      "theoretical_loss": 3.4370136897774968,
+      "tokens_seen": 1954283520
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001524257613013051,
+      "loss": 2.5468,
+      "theoretical_loss": 3.4368206964879313,
+      "tokens_seen": 1955594240
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001521893323245697,
+      "loss": 2.5599,
+      "theoretical_loss": 3.436627868698035,
+      "tokens_seen": 1956904960
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015195290334783432,
+      "loss": 2.5483,
+      "theoretical_loss": 3.436435206155176,
+      "tokens_seen": 1958215680
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015171647437109892,
+      "loss": 2.5516,
+      "theoretical_loss": 3.4362427086072795,
+      "tokens_seen": 1959526400
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001514800453943635,
+      "loss": 2.5419,
+      "theoretical_loss": 3.4360503758028216,
+      "tokens_seen": 1960837120
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015124361641762817,
+      "loss": 2.51,
+      "theoretical_loss": 3.4358582074908304,
+      "tokens_seen": 1962147840
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015100718744089276,
+      "loss": 2.5437,
+      "theoretical_loss": 3.4356662034208822,
+      "tokens_seen": 1963458560
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015077075846415736,
+      "loss": 2.5503,
+      "theoretical_loss": 3.4354743633431033,
+      "tokens_seen": 1964769280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4726476971055645,
+      "debugging/distinct-1-grams": 0.8065441543492692,
+      "debugging/distinct-2-grams": 0.9757863377628176,
+      "debugging/entropy-1-grams": 6.030268359568736,
+      "debugging/entropy-2-grams": 6.788237738678336,
+      "debugging/length": 462.57142857142856,
+      "debugging/num_segments": 14,
+      "epoch": 0.7,
+      "objective/train/docs_used": 1082459,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.3759894371032715,
+      "objective/train/theoretical_loss": 3.435282687008166,
+      "objective/train/tokens_used": 1986540000,
+      "theoretical_loss": 3.435282687008166,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015053432948742198,
+      "loss": 2.5449,
+      "theoretical_loss": 3.435282687008166,
+      "tokens_seen": 1966080000
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00015029790051068658,
+      "loss": 2.5482,
+      "theoretical_loss": 3.435091174167287,
+      "tokens_seen": 1967390720
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.0001500614715339512,
+      "loss": 2.5874,
+      "theoretical_loss": 3.4348998245722266,
+      "tokens_seen": 1968701440
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014982504255721583,
+      "loss": 2.5887,
+      "theoretical_loss": 3.4347086379752874,
+      "tokens_seen": 1970012160
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014958861358048043,
+      "loss": 2.5364,
+      "theoretical_loss": 3.4345176141293114,
+      "tokens_seen": 1971322880
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014935218460374503,
+      "loss": 2.636,
+      "theoretical_loss": 3.4343267527876806,
+      "tokens_seen": 1972633600
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00014911575562700965,
+      "loss": 2.6461,
+      "theoretical_loss": 3.4341360537043135,
+      "tokens_seen": 1973944320
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014887932665027425,
+      "loss": 2.6428,
+      "theoretical_loss": 3.4339455166336643,
+      "tokens_seen": 1975255040
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014864289767353887,
+      "loss": 2.6096,
+      "theoretical_loss": 3.433755141330721,
+      "tokens_seen": 1976565760
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001484064686968035,
+      "loss": 2.6226,
+      "theoretical_loss": 3.4335649275510063,
+      "tokens_seen": 1977876480
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001481700397200681,
+      "loss": 2.6145,
+      "theoretical_loss": 3.4333748750505717,
+      "tokens_seen": 1979187200
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001479336107433327,
+      "loss": 2.6281,
+      "theoretical_loss": 3.4331849835859996,
+      "tokens_seen": 1980497920
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014769718176659732,
+      "loss": 2.5908,
+      "theoretical_loss": 3.4329952529144,
+      "tokens_seen": 1981808640
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014746075278986194,
+      "loss": 2.6699,
+      "theoretical_loss": 3.4328056827934104,
+      "tokens_seen": 1983119360
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014722432381312654,
+      "loss": 2.6574,
+      "theoretical_loss": 3.4326162729811918,
+      "tokens_seen": 1984430080
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014698789483639116,
+      "loss": 2.6277,
+      "theoretical_loss": 3.4324270232364316,
+      "tokens_seen": 1985740800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014675146585965576,
+      "loss": 2.6691,
+      "theoretical_loss": 3.432237933318337,
+      "tokens_seen": 1987051520
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014651503688292036,
+      "loss": 2.6178,
+      "theoretical_loss": 3.432049002986637,
+      "tokens_seen": 1988362240
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014627860790618498,
+      "loss": 2.6073,
+      "theoretical_loss": 3.43186023200158,
+      "tokens_seen": 1989672960
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001460421789294496,
+      "loss": 2.6327,
+      "theoretical_loss": 3.4316716201239315,
+      "tokens_seen": 1990983680
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001458057499527142,
+      "loss": 2.6292,
+      "theoretical_loss": 3.431483167114974,
+      "tokens_seen": 1992294400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014556932097597883,
+      "loss": 2.6417,
+      "theoretical_loss": 3.4312948727365047,
+      "tokens_seen": 1993605120
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014533289199924343,
+      "loss": 2.6072,
+      "theoretical_loss": 3.4311067367508348,
+      "tokens_seen": 1994915840
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014509646302250802,
+      "loss": 2.6235,
+      "theoretical_loss": 3.4309187589207863,
+      "tokens_seen": 1996226560
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014486003404577268,
+      "loss": 2.6018,
+      "theoretical_loss": 3.430730939009693,
+      "tokens_seen": 1997537280
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014462360506903727,
+      "loss": 2.6088,
+      "theoretical_loss": 3.430543276781397,
+      "tokens_seen": 1998848000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00014438717609230187,
+      "loss": 2.5725,
+      "theoretical_loss": 3.430355772000249,
+      "tokens_seen": 2000158720
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001441507471155665,
+      "loss": 2.6084,
+      "theoretical_loss": 3.4301684244311046,
+      "tokens_seen": 2001469440
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001439143181388311,
+      "loss": 2.65,
+      "theoretical_loss": 3.4299812338393263,
+      "tokens_seen": 2002780160
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001436778891620957,
+      "loss": 2.5575,
+      "theoretical_loss": 3.429794199990778,
+      "tokens_seen": 2004090880
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014344146018536034,
+      "loss": 2.5852,
+      "theoretical_loss": 3.429607322651827,
+      "tokens_seen": 2005401600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014320503120862494,
+      "loss": 2.6265,
+      "theoretical_loss": 3.4294206015893405,
+      "tokens_seen": 2006712320
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014296860223188954,
+      "loss": 2.6129,
+      "theoretical_loss": 3.4292340365706853,
+      "tokens_seen": 2008023040
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014273217325515416,
+      "loss": 2.6118,
+      "theoretical_loss": 3.429047627363726,
+      "tokens_seen": 2009333760
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014249574427841876,
+      "loss": 2.6669,
+      "theoretical_loss": 3.4288613737368236,
+      "tokens_seen": 2010644480
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014225931530168338,
+      "loss": 2.5468,
+      "theoretical_loss": 3.428675275458835,
+      "tokens_seen": 2011955200
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014202288632494798,
+      "loss": 2.6063,
+      "theoretical_loss": 3.4284893322991095,
+      "tokens_seen": 2013265920
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001417864573482126,
+      "loss": 2.595,
+      "theoretical_loss": 3.4283035440274894,
+      "tokens_seen": 2014576640
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001415500283714772,
+      "loss": 2.6225,
+      "theoretical_loss": 3.4281179104143074,
+      "tokens_seen": 2015887360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.0001413135993947418,
+      "loss": 2.5768,
+      "theoretical_loss": 3.427932431230387,
+      "tokens_seen": 2017198080
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014107717041800645,
+      "loss": 2.5881,
+      "theoretical_loss": 3.427747106247039,
+      "tokens_seen": 2018508800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014084074144127105,
+      "loss": 2.614,
+      "theoretical_loss": 3.427561935236061,
+      "tokens_seen": 2019819520
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014060431246453565,
+      "loss": 2.5859,
+      "theoretical_loss": 3.4273769179697364,
+      "tokens_seen": 2021130240
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014036788348780027,
+      "loss": 2.5672,
+      "theoretical_loss": 3.4271920542208334,
+      "tokens_seen": 2022440960
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00014013145451106487,
+      "loss": 2.5308,
+      "theoretical_loss": 3.4270073437626016,
+      "tokens_seen": 2023751680
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013989502553432947,
+      "loss": 2.5544,
+      "theoretical_loss": 3.4268227863687732,
+      "tokens_seen": 2025062400
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013965859655759412,
+      "loss": 2.5477,
+      "theoretical_loss": 3.4266383818135604,
+      "tokens_seen": 2026373120
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013942216758085872,
+      "loss": 2.5937,
+      "theoretical_loss": 3.426454129871654,
+      "tokens_seen": 2027683840
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00013918573860412331,
+      "loss": 2.5375,
+      "theoretical_loss": 3.4262700303182223,
+      "tokens_seen": 2028994560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013894930962738794,
+      "loss": 2.5304,
+      "theoretical_loss": 3.42608608292891,
+      "tokens_seen": 2030305280
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 1119468,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.6160428524017334,
+      "objective/train/theoretical_loss": 3.4259022874798366,
+      "objective/train/tokens_used": 2052076000,
+      "theoretical_loss": 3.4259022874798366,
+      "tokens_seen": 2031616000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013871288065065254,
+      "loss": 2.5891,
+      "theoretical_loss": 3.4259022874798366,
+      "tokens_seen": 2031616000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013847645167391716,
+      "loss": 2.5509,
+      "theoretical_loss": 3.425718643747595,
+      "tokens_seen": 2032926720
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013824002269718179,
+      "loss": 2.6152,
+      "theoretical_loss": 3.4255351515092505,
+      "tokens_seen": 2034237440
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013800359372044638,
+      "loss": 2.6069,
+      "theoretical_loss": 3.42535181054234,
+      "tokens_seen": 2035548160
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013776716474371098,
+      "loss": 2.5641,
+      "theoretical_loss": 3.425168620624868,
+      "tokens_seen": 2036858880
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001375307357669756,
+      "loss": 2.614,
+      "theoretical_loss": 3.42498558153531,
+      "tokens_seen": 2038169600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001372943067902402,
+      "loss": 2.559,
+      "theoretical_loss": 3.424802693052607,
+      "tokens_seen": 2039480320
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013705787781350483,
+      "loss": 2.6533,
+      "theoretical_loss": 3.4246199549561664,
+      "tokens_seen": 2040791040
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013682144883676945,
+      "loss": 2.6017,
+      "theoretical_loss": 3.4244373670258597,
+      "tokens_seen": 2042101760
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013658501986003405,
+      "loss": 2.5826,
+      "theoretical_loss": 3.424254929042021,
+      "tokens_seen": 2043412480
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013634859088329865,
+      "loss": 2.6157,
+      "theoretical_loss": 3.4240726407854485,
+      "tokens_seen": 2044723200
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013611216190656327,
+      "loss": 2.6253,
+      "theoretical_loss": 3.4238905020373984,
+      "tokens_seen": 2046033920
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001358757329298279,
+      "loss": 2.63,
+      "theoretical_loss": 3.423708512579589,
+      "tokens_seen": 2047344640
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001356393039530925,
+      "loss": 2.5767,
+      "theoretical_loss": 3.423526672194195,
+      "tokens_seen": 2048655360
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013540287497635712,
+      "loss": 2.6764,
+      "theoretical_loss": 3.423344980663848,
+      "tokens_seen": 2049966080
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013516644599962171,
+      "loss": 2.6236,
+      "theoretical_loss": 3.4231634377716365,
+      "tokens_seen": 2051276800
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.0001349300170228863,
+      "loss": 2.6579,
+      "theoretical_loss": 3.422982043301102,
+      "tokens_seen": 2052587520
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013469358804615094,
+      "loss": 2.6525,
+      "theoretical_loss": 3.4228007970362406,
+      "tokens_seen": 2053898240
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013445715906941556,
+      "loss": 2.6433,
+      "theoretical_loss": 3.4226196987614994,
+      "tokens_seen": 2055208960
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013422073009268016,
+      "loss": 2.5677,
+      "theoretical_loss": 3.422438748261776,
+      "tokens_seen": 2056519680
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00013398430111594478,
+      "loss": 2.6073,
+      "theoretical_loss": 3.4222579453224187,
+      "tokens_seen": 2057830400
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013374787213920938,
+      "loss": 2.6096,
+      "theoretical_loss": 3.422077289729223,
+      "tokens_seen": 2059141120
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013351144316247398,
+      "loss": 2.6635,
+      "theoretical_loss": 3.4218967812684316,
+      "tokens_seen": 2060451840
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013327501418573863,
+      "loss": 2.6283,
+      "theoretical_loss": 3.4217164197267333,
+      "tokens_seen": 2061762560
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013303858520900323,
+      "loss": 2.6398,
+      "theoretical_loss": 3.421536204891262,
+      "tokens_seen": 2063073280
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013280215623226783,
+      "loss": 2.6608,
+      "theoretical_loss": 3.4213561365495933,
+      "tokens_seen": 2064384000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013256572725553245,
+      "loss": 2.6087,
+      "theoretical_loss": 3.4211762144897464,
+      "tokens_seen": 2065694720
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013232929827879705,
+      "loss": 2.6523,
+      "theoretical_loss": 3.4209964385001816,
+      "tokens_seen": 2067005440
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013209286930206164,
+      "loss": 2.6026,
+      "theoretical_loss": 3.4208168083697985,
+      "tokens_seen": 2068316160
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001318564403253263,
+      "loss": 2.5851,
+      "theoretical_loss": 3.4206373238879353,
+      "tokens_seen": 2069626880
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001316200113485909,
+      "loss": 2.5479,
+      "theoretical_loss": 3.4204579848443677,
+      "tokens_seen": 2070937600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001313835823718555,
+      "loss": 2.592,
+      "theoretical_loss": 3.420278791029308,
+      "tokens_seen": 2072248320
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013114715339512012,
+      "loss": 2.577,
+      "theoretical_loss": 3.4200997422334027,
+      "tokens_seen": 2073559040
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001309107244183847,
+      "loss": 2.6008,
+      "theoretical_loss": 3.4199208382477333,
+      "tokens_seen": 2074869760
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013067429544164934,
+      "loss": 2.55,
+      "theoretical_loss": 3.4197420788638127,
+      "tokens_seen": 2076180480
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013043786646491394,
+      "loss": 2.6227,
+      "theoretical_loss": 3.419563463873587,
+      "tokens_seen": 2077491200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00013020143748817856,
+      "loss": 2.56,
+      "theoretical_loss": 3.419384993069431,
+      "tokens_seen": 2078801920
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012996500851144316,
+      "loss": 2.54,
+      "theoretical_loss": 3.4192066662441496,
+      "tokens_seen": 2080112640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.00012972857953470775,
+      "loss": 2.5199,
+      "theoretical_loss": 3.419028483190976,
+      "tokens_seen": 2081423360
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001294921505579724,
+      "loss": 2.5221,
+      "theoretical_loss": 3.4188504437035694,
+      "tokens_seen": 2082734080
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.000129255721581237,
+      "loss": 2.6013,
+      "theoretical_loss": 3.4186725475760156,
+      "tokens_seen": 2084044800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 0.0001290192926045016,
+      "loss": 2.5747,
+      "theoretical_loss": 3.4184947946028243,
+      "tokens_seen": 2085355520
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012878286362776623,
+      "loss": 2.5078,
+      "theoretical_loss": 3.4183171845789295,
+      "tokens_seen": 2086666240
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012854643465103082,
+      "loss": 2.5495,
+      "theoretical_loss": 3.418139717299687,
+      "tokens_seen": 2087976960
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012831000567429542,
+      "loss": 2.5125,
+      "theoretical_loss": 3.4179623925608733,
+      "tokens_seen": 2089287680
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012807357669756007,
+      "loss": 2.5545,
+      "theoretical_loss": 3.4177852101586863,
+      "tokens_seen": 2090598400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012783714772082467,
+      "loss": 2.5834,
+      "theoretical_loss": 3.417608169889742,
+      "tokens_seen": 2091909120
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012760071874408927,
+      "loss": 2.5424,
+      "theoretical_loss": 3.4174312715510737,
+      "tokens_seen": 2093219840
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001273642897673539,
+      "loss": 2.5454,
+      "theoretical_loss": 3.417254514940132,
+      "tokens_seen": 2094530560
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001271278607906185,
+      "loss": 2.4824,
+      "theoretical_loss": 3.417077899854784,
+      "tokens_seen": 2095841280
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 1156276,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.489689826965332,
+      "objective/train/theoretical_loss": 3.4169014260933093,
+      "objective/train/tokens_used": 2117612000,
+      "theoretical_loss": 3.4169014260933093,
+      "tokens_seen": 2097152000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012689143181388311,
+      "loss": 2.576,
+      "theoretical_loss": 3.4169014260933093,
+      "tokens_seen": 2097152000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012665500283714774,
+      "loss": 2.5583,
+      "theoretical_loss": 3.416725093454403,
+      "tokens_seen": 2098462720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012641857386041234,
+      "loss": 2.6127,
+      "theoretical_loss": 3.41654890173717,
+      "tokens_seen": 2099773440
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012618214488367693,
+      "loss": 2.5472,
+      "theoretical_loss": 3.416372850741128,
+      "tokens_seen": 2101084160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012594571590694156,
+      "loss": 2.5435,
+      "theoretical_loss": 3.4161969402662056,
+      "tokens_seen": 2102394880
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012570928693020616,
+      "loss": 2.583,
+      "theoretical_loss": 3.416021170112738,
+      "tokens_seen": 2103705600
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012547285795347078,
+      "loss": 2.5108,
+      "theoretical_loss": 3.41584554008147,
+      "tokens_seen": 2105016320
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.0001252364289767354,
+      "loss": 2.5647,
+      "theoretical_loss": 3.4156700499735524,
+      "tokens_seen": 2106327040
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.000125,
+      "loss": 2.5662,
+      "theoretical_loss": 3.415494699590542,
+      "tokens_seen": 2107637760
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012476357102326463,
+      "loss": 2.509,
+      "theoretical_loss": 3.4153194887344007,
+      "tokens_seen": 2108948480
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012452714204652922,
+      "loss": 2.5412,
+      "theoretical_loss": 3.415144417207493,
+      "tokens_seen": 2110259200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012429071306979385,
+      "loss": 2.5345,
+      "theoretical_loss": 3.414969484812586,
+      "tokens_seen": 2111569920
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 0.00012405428409305845,
+      "loss": 2.5617,
+      "theoretical_loss": 3.4147946913528493,
+      "tokens_seen": 2112880640
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012381785511632304,
+      "loss": 2.5441,
+      "theoretical_loss": 3.414620036631852,
+      "tokens_seen": 2114191360
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012358142613958767,
+      "loss": 2.5456,
+      "theoretical_loss": 3.4144455204535618,
+      "tokens_seen": 2115502080
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001233449971628523,
+      "loss": 2.5329,
+      "theoretical_loss": 3.414271142622346,
+      "tokens_seen": 2116812800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001231085681861169,
+      "loss": 2.526,
+      "theoretical_loss": 3.414096902942968,
+      "tokens_seen": 2118123520
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012287213920938152,
+      "loss": 2.5466,
+      "theoretical_loss": 3.4139228012205884,
+      "tokens_seen": 2119434240
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001226357102326461,
+      "loss": 2.5338,
+      "theoretical_loss": 3.413748837260762,
+      "tokens_seen": 2120744960
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012239928125591074,
+      "loss": 2.5887,
+      "theoretical_loss": 3.413575010869437,
+      "tokens_seen": 2122055680
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012216285227917533,
+      "loss": 2.5429,
+      "theoretical_loss": 3.413401321852957,
+      "tokens_seen": 2123366400
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012192642330243995,
+      "loss": 2.5737,
+      "theoretical_loss": 3.413227770018054,
+      "tokens_seen": 2124677120
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012168999432570456,
+      "loss": 2.5509,
+      "theoretical_loss": 3.4130543551718544,
+      "tokens_seen": 2125987840
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012145356534896918,
+      "loss": 2.5721,
+      "theoretical_loss": 3.412881077121873,
+      "tokens_seen": 2127298560
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012121713637223378,
+      "loss": 2.5755,
+      "theoretical_loss": 3.4127079356760124,
+      "tokens_seen": 2128609280
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012098070739549839,
+      "loss": 2.572,
+      "theoretical_loss": 3.4125349306425647,
+      "tokens_seen": 2129920000
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012074427841876301,
+      "loss": 2.5373,
+      "theoretical_loss": 3.4123620618302075,
+      "tokens_seen": 2131230720
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012050784944202761,
+      "loss": 2.5926,
+      "theoretical_loss": 3.412189329048006,
+      "tokens_seen": 2132541440
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012027142046529222,
+      "loss": 2.609,
+      "theoretical_loss": 3.412016732105408,
+      "tokens_seen": 2133852160
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00012003499148855685,
+      "loss": 2.5978,
+      "theoretical_loss": 3.411844270812246,
+      "tokens_seen": 2135162880
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011979856251182145,
+      "loss": 2.624,
+      "theoretical_loss": 3.4116719449787354,
+      "tokens_seen": 2136473600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011956213353508606,
+      "loss": 2.5737,
+      "theoretical_loss": 3.411499754415474,
+      "tokens_seen": 2137784320
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011932570455835068,
+      "loss": 2.6009,
+      "theoretical_loss": 3.4113276989334387,
+      "tokens_seen": 2139095040
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.00011908927558161528,
+      "loss": 2.5577,
+      "theoretical_loss": 3.4111557783439874,
+      "tokens_seen": 2140405760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0001188528466048799,
+      "loss": 2.5786,
+      "theoretical_loss": 3.4109839924588563,
+      "tokens_seen": 2141716480
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011861641762814451,
+      "loss": 2.5465,
+      "theoretical_loss": 3.4108123410901596,
+      "tokens_seen": 2143027200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011837998865140911,
+      "loss": 2.5951,
+      "theoretical_loss": 3.4106408240503887,
+      "tokens_seen": 2144337920
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011814355967467374,
+      "loss": 2.6044,
+      "theoretical_loss": 3.410469441152409,
+      "tokens_seen": 2145648640
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011790713069793835,
+      "loss": 2.5803,
+      "theoretical_loss": 3.4102981922094626,
+      "tokens_seen": 2146959360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011767070172120294,
+      "loss": 2.5853,
+      "theoretical_loss": 3.4101270770351646,
+      "tokens_seen": 2148270080
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011743427274446757,
+      "loss": 2.5698,
+      "theoretical_loss": 3.409956095443503,
+      "tokens_seen": 2149580800
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011719784376773218,
+      "loss": 2.6108,
+      "theoretical_loss": 3.4097852472488386,
+      "tokens_seen": 2150891520
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011696141479099678,
+      "loss": 2.531,
+      "theoretical_loss": 3.409614532265901,
+      "tokens_seen": 2152202240
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001167249858142614,
+      "loss": 2.5992,
+      "theoretical_loss": 3.409443950309792,
+      "tokens_seen": 2153512960
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011648855683752601,
+      "loss": 2.5506,
+      "theoretical_loss": 3.4092735011959814,
+      "tokens_seen": 2154823680
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011625212786079062,
+      "loss": 2.659,
+      "theoretical_loss": 3.4091031847403066,
+      "tokens_seen": 2156134400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011601569888405524,
+      "loss": 2.592,
+      "theoretical_loss": 3.4089330007589727,
+      "tokens_seen": 2157445120
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011577926990731983,
+      "loss": 2.6023,
+      "theoretical_loss": 3.4087629490685503,
+      "tokens_seen": 2158755840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011554284093058446,
+      "loss": 2.6093,
+      "theoretical_loss": 3.408593029485976,
+      "tokens_seen": 2160066560
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011530641195384907,
+      "loss": 2.5717,
+      "theoretical_loss": 3.40842324182855,
+      "tokens_seen": 2161377280
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 1193371,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.717996120452881,
+      "objective/train/theoretical_loss": 3.4082535859139362,
+      "objective/train/tokens_used": 2183148000,
+      "theoretical_loss": 3.4082535859139362,
+      "tokens_seen": 2162688000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011506998297711367,
+      "loss": 2.6123,
+      "theoretical_loss": 3.4082535859139362,
+      "tokens_seen": 2162688000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011483355400037829,
+      "loss": 2.5808,
+      "theoretical_loss": 3.4080840615601606,
+      "tokens_seen": 2163998720
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.0001145971250236429,
+      "loss": 2.6487,
+      "theoretical_loss": 3.40791466858561,
+      "tokens_seen": 2165309440
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011436069604690751,
+      "loss": 2.6373,
+      "theoretical_loss": 3.407745406809033,
+      "tokens_seen": 2166620160
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011412426707017212,
+      "loss": 2.65,
+      "theoretical_loss": 3.407576276049536,
+      "tokens_seen": 2167930880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 0.00011388783809343673,
+      "loss": 2.6188,
+      "theoretical_loss": 3.4074072761265852,
+      "tokens_seen": 2169241600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011365140911670135,
+      "loss": 2.5943,
+      "theoretical_loss": 3.4072384068600043,
+      "tokens_seen": 2170552320
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011341498013996596,
+      "loss": 2.59,
+      "theoretical_loss": 3.407069668069974,
+      "tokens_seen": 2171863040
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011317855116323057,
+      "loss": 2.5873,
+      "theoretical_loss": 3.406901059577029,
+      "tokens_seen": 2173173760
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011294212218649518,
+      "loss": 2.5802,
+      "theoretical_loss": 3.406732581202062,
+      "tokens_seen": 2174484480
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011270569320975979,
+      "loss": 2.5736,
+      "theoretical_loss": 3.4065642327663173,
+      "tokens_seen": 2175795200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.0001124692642330244,
+      "loss": 2.6034,
+      "theoretical_loss": 3.406396014091392,
+      "tokens_seen": 2177105920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011223283525628901,
+      "loss": 2.5596,
+      "theoretical_loss": 3.4062279249992375,
+      "tokens_seen": 2178416640
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011199640627955362,
+      "loss": 2.5533,
+      "theoretical_loss": 3.406059965312155,
+      "tokens_seen": 2179727360
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011175997730281825,
+      "loss": 2.5976,
+      "theoretical_loss": 3.405892134852796,
+      "tokens_seen": 2181038080
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011152354832608284,
+      "loss": 2.5582,
+      "theoretical_loss": 3.4057244334441616,
+      "tokens_seen": 2182348800
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011128711934934746,
+      "loss": 2.5334,
+      "theoretical_loss": 3.405556860909602,
+      "tokens_seen": 2183659520
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011105069037261208,
+      "loss": 2.58,
+      "theoretical_loss": 3.4053894170728145,
+      "tokens_seen": 2184970240
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011081426139587668,
+      "loss": 2.5634,
+      "theoretical_loss": 3.4052221017578432,
+      "tokens_seen": 2186280960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011057783241914129,
+      "loss": 2.5487,
+      "theoretical_loss": 3.405054914789078,
+      "tokens_seen": 2187591680
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011034140344240591,
+      "loss": 2.5771,
+      "theoretical_loss": 3.4048878559912543,
+      "tokens_seen": 2188902400
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00011010497446567051,
+      "loss": 2.5733,
+      "theoretical_loss": 3.4047209251894515,
+      "tokens_seen": 2190213120
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010986854548893514,
+      "loss": 2.5745,
+      "theoretical_loss": 3.404554122209091,
+      "tokens_seen": 2191523840
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010963211651219973,
+      "loss": 2.5395,
+      "theoretical_loss": 3.4043874468759383,
+      "tokens_seen": 2192834560
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010939568753546434,
+      "loss": 2.517,
+      "theoretical_loss": 3.4042208990160994,
+      "tokens_seen": 2194145280
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010915925855872897,
+      "loss": 2.5561,
+      "theoretical_loss": 3.404054478456021,
+      "tokens_seen": 2195456000
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 0.00010892282958199357,
+      "loss": 2.548,
+      "theoretical_loss": 3.4038881850224896,
+      "tokens_seen": 2196766720
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010868640060525818,
+      "loss": 2.5406,
+      "theoretical_loss": 3.4037220185426307,
+      "tokens_seen": 2198077440
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001084499716285228,
+      "loss": 2.5458,
+      "theoretical_loss": 3.4035559788439076,
+      "tokens_seen": 2199388160
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001082135426517874,
+      "loss": 2.5656,
+      "theoretical_loss": 3.4033900657541207,
+      "tokens_seen": 2200698880
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010797711367505201,
+      "loss": 2.5648,
+      "theoretical_loss": 3.4032242791014067,
+      "tokens_seen": 2202009600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010774068469831663,
+      "loss": 2.5685,
+      "theoretical_loss": 3.4030586187142378,
+      "tokens_seen": 2203320320
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010750425572158123,
+      "loss": 2.6157,
+      "theoretical_loss": 3.4028930844214207,
+      "tokens_seen": 2204631040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010726782674484586,
+      "loss": 2.5775,
+      "theoretical_loss": 3.4027276760520957,
+      "tokens_seen": 2205941760
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010703139776811047,
+      "loss": 2.601,
+      "theoretical_loss": 3.4025623934357365,
+      "tokens_seen": 2207252480
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010679496879137507,
+      "loss": 2.5211,
+      "theoretical_loss": 3.402397236402148,
+      "tokens_seen": 2208563200
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010655853981463969,
+      "loss": 2.5436,
+      "theoretical_loss": 3.4022322047814675,
+      "tokens_seen": 2209873920
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001063221108379043,
+      "loss": 2.5536,
+      "theoretical_loss": 3.4020672984041607,
+      "tokens_seen": 2211184640
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.0001060856818611689,
+      "loss": 2.5126,
+      "theoretical_loss": 3.4019025171010244,
+      "tokens_seen": 2212495360
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010584925288443352,
+      "loss": 2.5267,
+      "theoretical_loss": 3.4017378607031836,
+      "tokens_seen": 2213806080
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010561282390769813,
+      "loss": 2.5316,
+      "theoretical_loss": 3.401573329042091,
+      "tokens_seen": 2215116800
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010537639493096273,
+      "loss": 2.5369,
+      "theoretical_loss": 3.4014089219495265,
+      "tokens_seen": 2216427520
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010513996595422736,
+      "loss": 2.5734,
+      "theoretical_loss": 3.401244639257596,
+      "tokens_seen": 2217738240
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010490353697749197,
+      "loss": 2.5861,
+      "theoretical_loss": 3.401080480798731,
+      "tokens_seen": 2219048960
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010466710800075658,
+      "loss": 2.5762,
+      "theoretical_loss": 3.400916446405687,
+      "tokens_seen": 2220359680
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010443067902402119,
+      "loss": 2.564,
+      "theoretical_loss": 3.400752535911544,
+      "tokens_seen": 2221670400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010419425004728579,
+      "loss": 2.5679,
+      "theoretical_loss": 3.4005887491497035,
+      "tokens_seen": 2222981120
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010395782107055041,
+      "loss": 2.6214,
+      "theoretical_loss": 3.400425085953891,
+      "tokens_seen": 2224291840
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 0.00010372139209381502,
+      "loss": 2.6044,
+      "theoretical_loss": 3.400261546158152,
+      "tokens_seen": 2225602560
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010348496311707962,
+      "loss": 2.6313,
+      "theoretical_loss": 3.4000981295968526,
+      "tokens_seen": 2226913280
+    },
+    {
+      "epoch": 0.8,
+      "objective/train/docs_used": 1229520,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.192460775375366,
+      "objective/train/theoretical_loss": 3.3999348361046784,
+      "objective/train/tokens_used": 2248684000,
+      "theoretical_loss": 3.3999348361046784,
+      "tokens_seen": 2228224000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010324853414034424,
+      "loss": 2.6107,
+      "theoretical_loss": 3.3999348361046784,
+      "tokens_seen": 2228224000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010301210516360886,
+      "loss": 2.6186,
+      "theoretical_loss": 3.3997716655166346,
+      "tokens_seen": 2229534720
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010277567618687347,
+      "loss": 2.6058,
+      "theoretical_loss": 3.3996086176680436,
+      "tokens_seen": 2230845440
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010253924721013808,
+      "loss": 2.5939,
+      "theoretical_loss": 3.3994456923945453,
+      "tokens_seen": 2232156160
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010230281823340269,
+      "loss": 2.5325,
+      "theoretical_loss": 3.3992828895320963,
+      "tokens_seen": 2233466880
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001020663892566673,
+      "loss": 2.581,
+      "theoretical_loss": 3.3991202089169685,
+      "tokens_seen": 2234777600
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010182996027993191,
+      "loss": 2.5867,
+      "theoretical_loss": 3.398957650385749,
+      "tokens_seen": 2236088320
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010159353130319652,
+      "loss": 2.5334,
+      "theoretical_loss": 3.3987952137753386,
+      "tokens_seen": 2237399040
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010135710232646113,
+      "loss": 2.5536,
+      "theoretical_loss": 3.3986328989229517,
+      "tokens_seen": 2238709760
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010112067334972574,
+      "loss": 2.5509,
+      "theoretical_loss": 3.398470705666115,
+      "tokens_seen": 2240020480
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010088424437299035,
+      "loss": 2.5661,
+      "theoretical_loss": 3.398308633842667,
+      "tokens_seen": 2241331200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010064781539625497,
+      "loss": 2.5541,
+      "theoretical_loss": 3.3981466832907565,
+      "tokens_seen": 2242641920
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.00010041138641951958,
+      "loss": 2.569,
+      "theoretical_loss": 3.3979848538488437,
+      "tokens_seen": 2243952640
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0001001749574427842,
+      "loss": 2.5918,
+      "theoretical_loss": 3.3978231453556975,
+      "tokens_seen": 2245263360
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.99385284660488e-05,
+      "loss": 2.5878,
+      "theoretical_loss": 3.397661557650395,
+      "tokens_seen": 2246574080
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.970209948931341e-05,
+      "loss": 2.5889,
+      "theoretical_loss": 3.397500090572322,
+      "tokens_seen": 2247884800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.946567051257803e-05,
+      "loss": 2.5989,
+      "theoretical_loss": 3.3973387439611704,
+      "tokens_seen": 2249195520
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.922924153584263e-05,
+      "loss": 2.5699,
+      "theoretical_loss": 3.3971775176569396,
+      "tokens_seen": 2250506240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.899281255910724e-05,
+      "loss": 2.5803,
+      "theoretical_loss": 3.397016411499933,
+      "tokens_seen": 2251816960
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 9.875638358237187e-05,
+      "loss": 2.5301,
+      "theoretical_loss": 3.3968554253307603,
+      "tokens_seen": 2253127680
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.851995460563646e-05,
+      "loss": 2.5577,
+      "theoretical_loss": 3.396694558990335,
+      "tokens_seen": 2254438400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.828352562890109e-05,
+      "loss": 2.6101,
+      "theoretical_loss": 3.396533812319872,
+      "tokens_seen": 2255749120
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.804709665216569e-05,
+      "loss": 2.6124,
+      "theoretical_loss": 3.3963731851608907,
+      "tokens_seen": 2257059840
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.78106676754303e-05,
+      "loss": 2.6465,
+      "theoretical_loss": 3.3962126773552126,
+      "tokens_seen": 2258370560
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.757423869869492e-05,
+      "loss": 2.6222,
+      "theoretical_loss": 3.3960522887449582,
+      "tokens_seen": 2259681280
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.733780972195952e-05,
+      "loss": 2.5878,
+      "theoretical_loss": 3.3958920191725497,
+      "tokens_seen": 2260992000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.710138074522413e-05,
+      "loss": 2.6049,
+      "theoretical_loss": 3.3957318684807087,
+      "tokens_seen": 2262302720
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.686495176848876e-05,
+      "loss": 2.6304,
+      "theoretical_loss": 3.3955718365124548,
+      "tokens_seen": 2263613440
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.662852279175335e-05,
+      "loss": 2.6937,
+      "theoretical_loss": 3.395411923111107,
+      "tokens_seen": 2264924160
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.639209381501796e-05,
+      "loss": 2.7399,
+      "theoretical_loss": 3.3952521281202794,
+      "tokens_seen": 2266234880
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.615566483828259e-05,
+      "loss": 2.8542,
+      "theoretical_loss": 3.3950924513838854,
+      "tokens_seen": 2267545600
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.591923586154719e-05,
+      "loss": 2.8068,
+      "theoretical_loss": 3.3949328927461324,
+      "tokens_seen": 2268856320
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.568280688481181e-05,
+      "loss": 2.7963,
+      "theoretical_loss": 3.394773452051523,
+      "tokens_seen": 2270167040
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.544637790807642e-05,
+      "loss": 2.8782,
+      "theoretical_loss": 3.394614129144855,
+      "tokens_seen": 2271477760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.520994893134102e-05,
+      "loss": 2.7615,
+      "theoretical_loss": 3.394454923871219,
+      "tokens_seen": 2272788480
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.497351995460564e-05,
+      "loss": 2.8244,
+      "theoretical_loss": 3.394295836075999,
+      "tokens_seen": 2274099200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.473709097787025e-05,
+      "loss": 2.79,
+      "theoretical_loss": 3.394136865604871,
+      "tokens_seen": 2275409920
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.450066200113485e-05,
+      "loss": 2.7675,
+      "theoretical_loss": 3.3939780123038026,
+      "tokens_seen": 2276720640
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.426423302439948e-05,
+      "loss": 2.7577,
+      "theoretical_loss": 3.393819276019052,
+      "tokens_seen": 2278031360
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.402780404766409e-05,
+      "loss": 2.7512,
+      "theoretical_loss": 3.393660656597168,
+      "tokens_seen": 2279342080
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.379137507092869e-05,
+      "loss": 2.7465,
+      "theoretical_loss": 3.3935021538849877,
+      "tokens_seen": 2280652800
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 9.355494609419331e-05,
+      "loss": 2.7839,
+      "theoretical_loss": 3.3933437677296374,
+      "tokens_seen": 2281963520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.331851711745792e-05,
+      "loss": 2.7241,
+      "theoretical_loss": 3.393185497978531,
+      "tokens_seen": 2283274240
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.308208814072253e-05,
+      "loss": 2.7145,
+      "theoretical_loss": 3.393027344479371,
+      "tokens_seen": 2284584960
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.284565916398714e-05,
+      "loss": 2.7544,
+      "theoretical_loss": 3.3928693070801437,
+      "tokens_seen": 2285895680
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.260923018725175e-05,
+      "loss": 2.6853,
+      "theoretical_loss": 3.392711385629124,
+      "tokens_seen": 2287206400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.237280121051636e-05,
+      "loss": 2.6751,
+      "theoretical_loss": 3.3925535799748703,
+      "tokens_seen": 2288517120
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.213637223378098e-05,
+      "loss": 2.6392,
+      "theoretical_loss": 3.392395889966225,
+      "tokens_seen": 2289827840
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.189994325704557e-05,
+      "loss": 2.614,
+      "theoretical_loss": 3.392238315452316,
+      "tokens_seen": 2291138560
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.16635142803102e-05,
+      "loss": 2.6672,
+      "theoretical_loss": 3.392080856282553,
+      "tokens_seen": 2292449280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.2570713966601559,
+      "debugging/distinct-1-grams": 0.7866229715323567,
+      "debugging/distinct-2-grams": 0.9423664826935307,
+      "debugging/entropy-1-grams": 4.8965841193538235,
+      "debugging/entropy-2-grams": 5.4353298290166965,
+      "debugging/length": 484.5,
+      "debugging/num_segments": 4,
+      "epoch": 0.82,
+      "objective/train/docs_used": 1259419,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.8271231651306152,
+      "objective/train/theoretical_loss": 3.391923512306627,
+      "objective/train/tokens_used": 2314220000,
+      "theoretical_loss": 3.391923512306627,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.142708530357481e-05,
+      "loss": 2.5621,
+      "theoretical_loss": 3.391923512306627,
+      "tokens_seen": 2293760000
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.119065632683942e-05,
+      "loss": 2.6429,
+      "theoretical_loss": 3.391766283374513,
+      "tokens_seen": 2295070720
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.095422735010403e-05,
+      "loss": 2.6107,
+      "theoretical_loss": 3.3916091693364647,
+      "tokens_seen": 2296381440
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.071779837336864e-05,
+      "loss": 2.6216,
+      "theoretical_loss": 3.391452170043018,
+      "tokens_seen": 2297692160
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.048136939663325e-05,
+      "loss": 2.629,
+      "theoretical_loss": 3.3912952853449867,
+      "tokens_seen": 2299002880
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.024494041989786e-05,
+      "loss": 2.5956,
+      "theoretical_loss": 3.391138515093464,
+      "tokens_seen": 2300313600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 9.000851144316248e-05,
+      "loss": 2.5682,
+      "theoretical_loss": 3.3909818591398224,
+      "tokens_seen": 2301624320
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.977208246642709e-05,
+      "loss": 2.5811,
+      "theoretical_loss": 3.39082531733571,
+      "tokens_seen": 2302935040
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.95356534896917e-05,
+      "loss": 2.5632,
+      "theoretical_loss": 3.390668889533053,
+      "tokens_seen": 2304245760
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.929922451295631e-05,
+      "loss": 2.6045,
+      "theoretical_loss": 3.3905125755840535,
+      "tokens_seen": 2305556480
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.906279553622092e-05,
+      "loss": 2.5469,
+      "theoretical_loss": 3.390356375341189,
+      "tokens_seen": 2306867200
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.882636655948553e-05,
+      "loss": 2.5495,
+      "theoretical_loss": 3.390200288657212,
+      "tokens_seen": 2308177920
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 8.858993758275015e-05,
+      "loss": 2.6182,
+      "theoretical_loss": 3.3900443153851496,
+      "tokens_seen": 2309488640
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.836533005485152e-05,
+      "loss": 2.5566,
+      "theoretical_loss": 3.389888455378301,
+      "tokens_seen": 2310799360
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.812890107811614e-05,
+      "loss": 2.542,
+      "theoretical_loss": 3.3897327084902393,
+      "tokens_seen": 2312110080
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.789247210138075e-05,
+      "loss": 2.5086,
+      "theoretical_loss": 3.3895770745748095,
+      "tokens_seen": 2313420800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.765604312464535e-05,
+      "loss": 2.5651,
+      "theoretical_loss": 3.3894215534861285,
+      "tokens_seen": 2314731520
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.741961414790997e-05,
+      "loss": 2.5698,
+      "theoretical_loss": 3.389266145078584,
+      "tokens_seen": 2316042240
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.718318517117459e-05,
+      "loss": 2.5487,
+      "theoretical_loss": 3.389110849206833,
+      "tokens_seen": 2317352960
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.694675619443918e-05,
+      "loss": 2.618,
+      "theoretical_loss": 3.3889556657258026,
+      "tokens_seen": 2318663680
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.671032721770381e-05,
+      "loss": 2.5664,
+      "theoretical_loss": 3.388800594490689,
+      "tokens_seen": 2319974400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.647389824096842e-05,
+      "loss": 2.5487,
+      "theoretical_loss": 3.3886456353569567,
+      "tokens_seen": 2321285120
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.623746926423303e-05,
+      "loss": 2.5624,
+      "theoretical_loss": 3.388490788180338,
+      "tokens_seen": 2322595840
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.600104028749764e-05,
+      "loss": 2.5153,
+      "theoretical_loss": 3.3883360528168307,
+      "tokens_seen": 2323906560
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.576461131076225e-05,
+      "loss": 2.563,
+      "theoretical_loss": 3.3881814291227004,
+      "tokens_seen": 2325217280
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.552818233402686e-05,
+      "loss": 2.5484,
+      "theoretical_loss": 3.3880269169544786,
+      "tokens_seen": 2326528000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.529175335729147e-05,
+      "loss": 2.5384,
+      "theoretical_loss": 3.38787251616896,
+      "tokens_seen": 2327838720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.505532438055607e-05,
+      "loss": 2.5532,
+      "theoretical_loss": 3.387718226623205,
+      "tokens_seen": 2329149440
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.48188954038207e-05,
+      "loss": 2.5343,
+      "theoretical_loss": 3.387564048174538,
+      "tokens_seen": 2330460160
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.458246642708531e-05,
+      "loss": 2.5737,
+      "theoretical_loss": 3.3874099806805456,
+      "tokens_seen": 2331770880
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.43460374503499e-05,
+      "loss": 2.5326,
+      "theoretical_loss": 3.387256023999078,
+      "tokens_seen": 2333081600
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.410960847361453e-05,
+      "loss": 2.6194,
+      "theoretical_loss": 3.387102177988246,
+      "tokens_seen": 2334392320
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.387317949687914e-05,
+      "loss": 2.5264,
+      "theoretical_loss": 3.386948442506422,
+      "tokens_seen": 2335703040
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.363675052014375e-05,
+      "loss": 2.5415,
+      "theoretical_loss": 3.3867948174122398,
+      "tokens_seen": 2337013760
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.340032154340836e-05,
+      "loss": 2.5137,
+      "theoretical_loss": 3.386641302564592,
+      "tokens_seen": 2338324480
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.316389256667297e-05,
+      "loss": 2.5726,
+      "theoretical_loss": 3.386487897822631,
+      "tokens_seen": 2339635200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.292746358993758e-05,
+      "loss": 2.5293,
+      "theoretical_loss": 3.386334603045769,
+      "tokens_seen": 2340945920
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.26910346132022e-05,
+      "loss": 2.572,
+      "theoretical_loss": 3.3861814180936745,
+      "tokens_seen": 2342256640
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.24546056364668e-05,
+      "loss": 2.5591,
+      "theoretical_loss": 3.386028342826274,
+      "tokens_seen": 2343567360
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.221817665973142e-05,
+      "loss": 2.5648,
+      "theoretical_loss": 3.3858753771037517,
+      "tokens_seen": 2344878080
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.198174768299603e-05,
+      "loss": 2.5973,
+      "theoretical_loss": 3.3857225207865476,
+      "tokens_seen": 2346188800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.174531870626065e-05,
+      "loss": 2.5962,
+      "theoretical_loss": 3.3855697737353565,
+      "tokens_seen": 2347499520
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.150888972952525e-05,
+      "loss": 2.5362,
+      "theoretical_loss": 3.38541713581113,
+      "tokens_seen": 2348810240
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.127246075278986e-05,
+      "loss": 2.602,
+      "theoretical_loss": 3.3852646068750722,
+      "tokens_seen": 2350120960
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.103603177605449e-05,
+      "loss": 2.6456,
+      "theoretical_loss": 3.3851121867886422,
+      "tokens_seen": 2351431680
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.079960279931908e-05,
+      "loss": 2.6091,
+      "theoretical_loss": 3.384959875413552,
+      "tokens_seen": 2352742400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.05631738225837e-05,
+      "loss": 2.5779,
+      "theoretical_loss": 3.3848076726117666,
+      "tokens_seen": 2354053120
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.032674484584832e-05,
+      "loss": 2.6389,
+      "theoretical_loss": 3.3846555782455017,
+      "tokens_seen": 2355363840
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.009031586911292e-05,
+      "loss": 2.5954,
+      "theoretical_loss": 3.384503592177226,
+      "tokens_seen": 2356674560
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.985388689237753e-05,
+      "loss": 2.6033,
+      "theoretical_loss": 3.3843517142696586,
+      "tokens_seen": 2357985280
+    },
+    {
+      "epoch": 0.84,
+      "objective/train/docs_used": 1294624,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.2581777572631836,
+      "objective/train/theoretical_loss": 3.3841999443857675,
+      "objective/train/tokens_used": 2379756000,
+      "theoretical_loss": 3.3841999443857675,
+      "tokens_seen": 2359296000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.961745791564215e-05,
+      "loss": 2.6283,
+      "theoretical_loss": 3.3841999443857675,
+      "tokens_seen": 2359296000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.938102893890675e-05,
+      "loss": 2.6429,
+      "theoretical_loss": 3.3840482823887728,
+      "tokens_seen": 2360606720
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.914459996217137e-05,
+      "loss": 2.7246,
+      "theoretical_loss": 3.3838967281421404,
+      "tokens_seen": 2361917440
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.890817098543597e-05,
+      "loss": 2.784,
+      "theoretical_loss": 3.3837452815095874,
+      "tokens_seen": 2363228160
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.867174200870058e-05,
+      "loss": 2.7861,
+      "theoretical_loss": 3.383593942355078,
+      "tokens_seen": 2364538880
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 7.843531303196521e-05,
+      "loss": 2.7483,
+      "theoretical_loss": 3.383442710542823,
+      "tokens_seen": 2365849600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.81988840552298e-05,
+      "loss": 2.7243,
+      "theoretical_loss": 3.38329158593728,
+      "tokens_seen": 2367160320
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.796245507849442e-05,
+      "loss": 2.6775,
+      "theoretical_loss": 3.383140568403153,
+      "tokens_seen": 2368471040
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.772602610175904e-05,
+      "loss": 2.6856,
+      "theoretical_loss": 3.3829896578053917,
+      "tokens_seen": 2369781760
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.748959712502364e-05,
+      "loss": 2.7022,
+      "theoretical_loss": 3.38283885400919,
+      "tokens_seen": 2371092480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.725316814828825e-05,
+      "loss": 2.645,
+      "theoretical_loss": 3.382688156879987,
+      "tokens_seen": 2372403200
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.701673917155287e-05,
+      "loss": 2.6439,
+      "theoretical_loss": 3.3825375662834647,
+      "tokens_seen": 2373713920
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.678031019481747e-05,
+      "loss": 2.64,
+      "theoretical_loss": 3.3823870820855495,
+      "tokens_seen": 2375024640
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.65438812180821e-05,
+      "loss": 2.6274,
+      "theoretical_loss": 3.3822367041524086,
+      "tokens_seen": 2376335360
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.63074522413467e-05,
+      "loss": 2.6278,
+      "theoretical_loss": 3.382086432350453,
+      "tokens_seen": 2377646080
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.60710232646113e-05,
+      "loss": 2.5777,
+      "theoretical_loss": 3.381936266546334,
+      "tokens_seen": 2378956800
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.583459428787593e-05,
+      "loss": 2.6042,
+      "theoretical_loss": 3.3817862066069444,
+      "tokens_seen": 2380267520
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.559816531114054e-05,
+      "loss": 2.5805,
+      "theoretical_loss": 3.3816362523994172,
+      "tokens_seen": 2381578240
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.536173633440514e-05,
+      "loss": 2.6919,
+      "theoretical_loss": 3.381486403791125,
+      "tokens_seen": 2382888960
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.512530735766976e-05,
+      "loss": 2.6666,
+      "theoretical_loss": 3.38133666064968,
+      "tokens_seen": 2384199680
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.488887838093437e-05,
+      "loss": 2.5689,
+      "theoretical_loss": 3.381187022842932,
+      "tokens_seen": 2385510400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.465244940419898e-05,
+      "loss": 2.5689,
+      "theoretical_loss": 3.381037490238971,
+      "tokens_seen": 2386821120
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.44160204274636e-05,
+      "loss": 2.6207,
+      "theoretical_loss": 3.3808880627061213,
+      "tokens_seen": 2388131840
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.41795914507282e-05,
+      "loss": 2.6122,
+      "theoretical_loss": 3.3807387401129474,
+      "tokens_seen": 2389442560
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.394316247399282e-05,
+      "loss": 2.6156,
+      "theoretical_loss": 3.3805895223282487,
+      "tokens_seen": 2390753280
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.370673349725743e-05,
+      "loss": 2.5663,
+      "theoretical_loss": 3.38044040922106,
+      "tokens_seen": 2392064000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.347030452052203e-05,
+      "loss": 2.5695,
+      "theoretical_loss": 3.380291400660652,
+      "tokens_seen": 2393374720
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.323387554378665e-05,
+      "loss": 2.5719,
+      "theoretical_loss": 3.3801424965165303,
+      "tokens_seen": 2394685440
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.299744656705126e-05,
+      "loss": 2.5775,
+      "theoretical_loss": 3.3799936966584343,
+      "tokens_seen": 2395996160
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.276101759031586e-05,
+      "loss": 2.6004,
+      "theoretical_loss": 3.3798450009563377,
+      "tokens_seen": 2397306880
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.252458861358048e-05,
+      "loss": 2.592,
+      "theoretical_loss": 3.379696409280446,
+      "tokens_seen": 2398617600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.22881596368451e-05,
+      "loss": 2.5584,
+      "theoretical_loss": 3.379547921501199,
+      "tokens_seen": 2399928320
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.20517306601097e-05,
+      "loss": 2.5238,
+      "theoretical_loss": 3.379399537489267,
+      "tokens_seen": 2401239040
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.181530168337432e-05,
+      "loss": 2.6113,
+      "theoretical_loss": 3.379251257115553,
+      "tokens_seen": 2402549760
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.157887270663893e-05,
+      "loss": 2.5544,
+      "theoretical_loss": 3.3791030802511894,
+      "tokens_seen": 2403860480
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.134244372990354e-05,
+      "loss": 2.5805,
+      "theoretical_loss": 3.3789550067675407,
+      "tokens_seen": 2405171200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.110601475316815e-05,
+      "loss": 2.6073,
+      "theoretical_loss": 3.3788070365361995,
+      "tokens_seen": 2406481920
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.086958577643276e-05,
+      "loss": 2.5353,
+      "theoretical_loss": 3.37865916942899,
+      "tokens_seen": 2407792640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.063315679969737e-05,
+      "loss": 2.635,
+      "theoretical_loss": 3.3785114053179623,
+      "tokens_seen": 2409103360
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.039672782296198e-05,
+      "loss": 2.5982,
+      "theoretical_loss": 3.378363744075397,
+      "tokens_seen": 2410414080
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.01602988462266e-05,
+      "loss": 2.6122,
+      "theoretical_loss": 3.3782161855738027,
+      "tokens_seen": 2411724800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.99238698694912e-05,
+      "loss": 2.59,
+      "theoretical_loss": 3.3780687296859124,
+      "tokens_seen": 2413035520
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.968744089275582e-05,
+      "loss": 2.5971,
+      "theoretical_loss": 3.3779213762846885,
+      "tokens_seen": 2414346240
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.945101191602044e-05,
+      "loss": 2.5529,
+      "theoretical_loss": 3.3777741252433184,
+      "tokens_seen": 2415656960
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.921458293928504e-05,
+      "loss": 2.5843,
+      "theoretical_loss": 3.3776269764352156,
+      "tokens_seen": 2416967680
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.897815396254965e-05,
+      "loss": 2.6258,
+      "theoretical_loss": 3.3774799297340183,
+      "tokens_seen": 2418278400
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.874172498581427e-05,
+      "loss": 2.6583,
+      "theoretical_loss": 3.377332985013589,
+      "tokens_seen": 2419589120
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 6.850529600907887e-05,
+      "loss": 2.6895,
+      "theoretical_loss": 3.377186142148015,
+      "tokens_seen": 2420899840
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.826886703234348e-05,
+      "loss": 2.6944,
+      "theoretical_loss": 3.3770394010116065,
+      "tokens_seen": 2422210560
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.80324380556081e-05,
+      "loss": 2.5833,
+      "theoretical_loss": 3.376892761478897,
+      "tokens_seen": 2423521280
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1329420,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.621964454650879,
+      "objective/train/theoretical_loss": 3.376746223424642,
+      "objective/train/tokens_used": 2445292000,
+      "theoretical_loss": 3.376746223424642,
+      "tokens_seen": 2424832000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.77960090788727e-05,
+      "loss": 2.6472,
+      "theoretical_loss": 3.376746223424642,
+      "tokens_seen": 2424832000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.755958010213733e-05,
+      "loss": 2.6471,
+      "theoretical_loss": 3.376599786723821,
+      "tokens_seen": 2426142720
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.732315112540193e-05,
+      "loss": 2.6344,
+      "theoretical_loss": 3.3764534512516313,
+      "tokens_seen": 2427453440
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.708672214866654e-05,
+      "loss": 2.5831,
+      "theoretical_loss": 3.376307216883494,
+      "tokens_seen": 2428764160
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.685029317193116e-05,
+      "loss": 2.5889,
+      "theoretical_loss": 3.3761610834950506,
+      "tokens_seen": 2430074880
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.661386419519576e-05,
+      "loss": 2.5895,
+      "theoretical_loss": 3.376015050962161,
+      "tokens_seen": 2431385600
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.637743521846037e-05,
+      "loss": 2.5954,
+      "theoretical_loss": 3.3758691191609054,
+      "tokens_seen": 2432696320
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.6141006241725e-05,
+      "loss": 2.664,
+      "theoretical_loss": 3.375723287967583,
+      "tokens_seen": 2434007040
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.590457726498959e-05,
+      "loss": 2.5926,
+      "theoretical_loss": 3.375577557258711,
+      "tokens_seen": 2435317760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.56681482882542e-05,
+      "loss": 2.5661,
+      "theoretical_loss": 3.375431926911025,
+      "tokens_seen": 2436628480
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.543171931151883e-05,
+      "loss": 2.6206,
+      "theoretical_loss": 3.375286396801478,
+      "tokens_seen": 2437939200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.519529033478342e-05,
+      "loss": 2.5517,
+      "theoretical_loss": 3.3751409668072396,
+      "tokens_seen": 2439249920
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.495886135804805e-05,
+      "loss": 2.6218,
+      "theoretical_loss": 3.3749956368056955,
+      "tokens_seen": 2440560640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.472243238131266e-05,
+      "loss": 2.5894,
+      "theoretical_loss": 3.3748504066744482,
+      "tokens_seen": 2441871360
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.448600340457726e-05,
+      "loss": 2.6008,
+      "theoretical_loss": 3.374705276291315,
+      "tokens_seen": 2443182080
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.424957442784188e-05,
+      "loss": 2.6083,
+      "theoretical_loss": 3.374560245534328,
+      "tokens_seen": 2444492800
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.40131454511065e-05,
+      "loss": 2.5999,
+      "theoretical_loss": 3.374415314281735,
+      "tokens_seen": 2445803520
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.377671647437109e-05,
+      "loss": 2.6051,
+      "theoretical_loss": 3.374270482411995,
+      "tokens_seen": 2447114240
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.354028749763572e-05,
+      "loss": 2.6185,
+      "theoretical_loss": 3.3741257498037838,
+      "tokens_seen": 2448424960
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 6.330385852090033e-05,
+      "loss": 2.6473,
+      "theoretical_loss": 3.3739811163359876,
+      "tokens_seen": 2449735680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.306742954416494e-05,
+      "loss": 2.6443,
+      "theoretical_loss": 3.3738365818877067,
+      "tokens_seen": 2451046400
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.283100056742955e-05,
+      "loss": 2.5845,
+      "theoretical_loss": 3.3736921463382528,
+      "tokens_seen": 2452357120
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.259457159069416e-05,
+      "loss": 2.5577,
+      "theoretical_loss": 3.373547809567148,
+      "tokens_seen": 2453667840
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.236996406279554e-05,
+      "loss": 2.651,
+      "theoretical_loss": 3.3734035714541273,
+      "tokens_seen": 2454978560
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.213353508606015e-05,
+      "loss": 2.6638,
+      "theoretical_loss": 3.373259431879135,
+      "tokens_seen": 2456289280
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.189710610932476e-05,
+      "loss": 2.5982,
+      "theoretical_loss": 3.3731153907223264,
+      "tokens_seen": 2457600000
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.166067713258937e-05,
+      "loss": 2.5933,
+      "theoretical_loss": 3.3729714478640656,
+      "tokens_seen": 2458910720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.142424815585398e-05,
+      "loss": 2.569,
+      "theoretical_loss": 3.372827603184925,
+      "tokens_seen": 2460221440
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.118781917911859e-05,
+      "loss": 2.6017,
+      "theoretical_loss": 3.372683856565688,
+      "tokens_seen": 2461532160
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.09513902023832e-05,
+      "loss": 2.6105,
+      "theoretical_loss": 3.3725402078873445,
+      "tokens_seen": 2462842880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.071496122564782e-05,
+      "loss": 2.6127,
+      "theoretical_loss": 3.372396657031092,
+      "tokens_seen": 2464153600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0490353697749195e-05,
+      "loss": 2.6502,
+      "theoretical_loss": 3.3722532038783357,
+      "tokens_seen": 2465464320
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0253924721013806e-05,
+      "loss": 2.5649,
+      "theoretical_loss": 3.372109848310687,
+      "tokens_seen": 2466775040
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.0017495744278424e-05,
+      "loss": 2.5789,
+      "theoretical_loss": 3.371966590209965,
+      "tokens_seen": 2468085760
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.978106676754303e-05,
+      "loss": 2.5876,
+      "theoretical_loss": 3.3718234294581926,
+      "tokens_seen": 2469396480
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.954463779080764e-05,
+      "loss": 2.5324,
+      "theoretical_loss": 3.3716803659375993,
+      "tokens_seen": 2470707200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.930820881407226e-05,
+      "loss": 2.572,
+      "theoretical_loss": 3.371537399530619,
+      "tokens_seen": 2472017920
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.907177983733687e-05,
+      "loss": 2.6096,
+      "theoretical_loss": 3.37139453011989,
+      "tokens_seen": 2473328640
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.883535086060147e-05,
+      "loss": 2.4975,
+      "theoretical_loss": 3.3712517575882552,
+      "tokens_seen": 2474639360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.859892188386609e-05,
+      "loss": 2.5031,
+      "theoretical_loss": 3.3711090818187603,
+      "tokens_seen": 2475950080
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 5.83624929071307e-05,
+      "loss": 2.5392,
+      "theoretical_loss": 3.3709665026946536,
+      "tokens_seen": 2477260800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.812606393039531e-05,
+      "loss": 2.562,
+      "theoretical_loss": 3.370824020099387,
+      "tokens_seen": 2478571520
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7889634953659916e-05,
+      "loss": 2.5704,
+      "theoretical_loss": 3.3706816339166137,
+      "tokens_seen": 2479882240
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7653205976924534e-05,
+      "loss": 2.5527,
+      "theoretical_loss": 3.3705393440301883,
+      "tokens_seen": 2481192960
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7416777000189145e-05,
+      "loss": 2.554,
+      "theoretical_loss": 3.3703971503241683,
+      "tokens_seen": 2482503680
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.7180348023453756e-05,
+      "loss": 2.5921,
+      "theoretical_loss": 3.37025505268281,
+      "tokens_seen": 2483814400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.694391904671837e-05,
+      "loss": 2.5511,
+      "theoretical_loss": 3.3701130509905703,
+      "tokens_seen": 2485125120
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.670749006998298e-05,
+      "loss": 2.626,
+      "theoretical_loss": 3.369971145132107,
+      "tokens_seen": 2486435840
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.647106109324759e-05,
+      "loss": 2.5566,
+      "theoretical_loss": 3.3698293349922754,
+      "tokens_seen": 2487746560
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.62346321165122e-05,
+      "loss": 2.579,
+      "theoretical_loss": 3.3696876204561326,
+      "tokens_seen": 2489057280
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1363869,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.2189786434173584,
+      "objective/train/theoretical_loss": 3.369546001408931,
+      "objective/train/tokens_used": 2510828000,
+      "theoretical_loss": 3.369546001408931,
+      "tokens_seen": 2490368000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.599820313977681e-05,
+      "loss": 2.5838,
+      "theoretical_loss": 3.369546001408931,
+      "tokens_seen": 2490368000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.576177416304142e-05,
+      "loss": 2.6302,
+      "theoretical_loss": 3.369404477736123,
+      "tokens_seen": 2491678720
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.552534518630604e-05,
+      "loss": 2.5681,
+      "theoretical_loss": 3.3692630493233575,
+      "tokens_seen": 2492989440
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5288916209570644e-05,
+      "loss": 2.5646,
+      "theoretical_loss": 3.369121716056482,
+      "tokens_seen": 2494300160
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.5052487232835255e-05,
+      "loss": 2.5404,
+      "theoretical_loss": 3.3689804778215393,
+      "tokens_seen": 2495610880
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4816058256099866e-05,
+      "loss": 2.4974,
+      "theoretical_loss": 3.3688393345047682,
+      "tokens_seen": 2496921600
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.4579629279364484e-05,
+      "loss": 2.5574,
+      "theoretical_loss": 3.3686982859926053,
+      "tokens_seen": 2498232320
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.434320030262909e-05,
+      "loss": 2.5313,
+      "theoretical_loss": 3.3685573321716804,
+      "tokens_seen": 2499543040
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.41067713258937e-05,
+      "loss": 2.5718,
+      "theoretical_loss": 3.368416472928819,
+      "tokens_seen": 2500853760
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.387034234915832e-05,
+      "loss": 2.6006,
+      "theoretical_loss": 3.368275708151042,
+      "tokens_seen": 2502164480
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.363391337242293e-05,
+      "loss": 2.5536,
+      "theoretical_loss": 3.3681350377255623,
+      "tokens_seen": 2503475200
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.339748439568753e-05,
+      "loss": 2.5862,
+      "theoretical_loss": 3.367994461539789,
+      "tokens_seen": 2504785920
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.316105541895215e-05,
+      "loss": 2.5073,
+      "theoretical_loss": 3.367853979481322,
+      "tokens_seen": 2506096640
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.292462644221676e-05,
+      "loss": 2.5112,
+      "theoretical_loss": 3.3677135914379552,
+      "tokens_seen": 2507407360
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2688197465481366e-05,
+      "loss": 2.517,
+      "theoretical_loss": 3.367573297297675,
+      "tokens_seen": 2508718080
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2451768488745983e-05,
+      "loss": 2.5122,
+      "theoretical_loss": 3.367433096948659,
+      "tokens_seen": 2510028800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.2215339512010595e-05,
+      "loss": 2.5173,
+      "theoretical_loss": 3.3672929902792768,
+      "tokens_seen": 2511339520
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.199073198411197e-05,
+      "loss": 2.5282,
+      "theoretical_loss": 3.3671529771780886,
+      "tokens_seen": 2512650240
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.175430300737659e-05,
+      "loss": 2.5105,
+      "theoretical_loss": 3.3670130575338457,
+      "tokens_seen": 2513960960
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.15178740306412e-05,
+      "loss": 2.5953,
+      "theoretical_loss": 3.366873231235489,
+      "tokens_seen": 2515271680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.12814450539058e-05,
+      "loss": 2.5644,
+      "theoretical_loss": 3.3667334981721493,
+      "tokens_seen": 2516582400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.104501607717042e-05,
+      "loss": 2.5741,
+      "theoretical_loss": 3.3665938582331476,
+      "tokens_seen": 2517893120
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.080858710043503e-05,
+      "loss": 2.5531,
+      "theoretical_loss": 3.3664543113079923,
+      "tokens_seen": 2519203840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.057215812369964e-05,
+      "loss": 2.5338,
+      "theoretical_loss": 3.3663148572863815,
+      "tokens_seen": 2520514560
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.033572914696425e-05,
+      "loss": 2.549,
+      "theoretical_loss": 3.3661754960582013,
+      "tokens_seen": 2521825280
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.0099300170228865e-05,
+      "loss": 2.5315,
+      "theoretical_loss": 3.3660362275135247,
+      "tokens_seen": 2523136000
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.9862871193493476e-05,
+      "loss": 2.5345,
+      "theoretical_loss": 3.3658970515426123,
+      "tokens_seen": 2524446720
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.962644221675809e-05,
+      "loss": 2.5519,
+      "theoretical_loss": 3.365757968035912,
+      "tokens_seen": 2525757440
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.93900132400227e-05,
+      "loss": 2.5385,
+      "theoretical_loss": 3.365618976884058,
+      "tokens_seen": 2527068160
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.915358426328731e-05,
+      "loss": 2.5985,
+      "theoretical_loss": 3.3654800779778693,
+      "tokens_seen": 2528378880
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.891715528655192e-05,
+      "loss": 2.5977,
+      "theoretical_loss": 3.3653412712083526,
+      "tokens_seen": 2529689600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.868072630981654e-05,
+      "loss": 2.5391,
+      "theoretical_loss": 3.365202556466697,
+      "tokens_seen": 2531000320
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.844429733308114e-05,
+      "loss": 2.577,
+      "theoretical_loss": 3.3650639336442794,
+      "tokens_seen": 2532311040
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.8207868356345753e-05,
+      "loss": 2.5187,
+      "theoretical_loss": 3.364925402632659,
+      "tokens_seen": 2533621760
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7971439379610364e-05,
+      "loss": 2.5572,
+      "theoretical_loss": 3.3647869633235796,
+      "tokens_seen": 2534932480
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7735010402874975e-05,
+      "loss": 2.5633,
+      "theoretical_loss": 3.364648615608968,
+      "tokens_seen": 2536243200
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7498581426139587e-05,
+      "loss": 2.5277,
+      "theoretical_loss": 3.364510359380936,
+      "tokens_seen": 2537553920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.72621524494042e-05,
+      "loss": 2.6168,
+      "theoretical_loss": 3.364372194531775,
+      "tokens_seen": 2538864640
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.7025723472668815e-05,
+      "loss": 2.541,
+      "theoretical_loss": 3.3642341209539617,
+      "tokens_seen": 2540175360
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.678929449593342e-05,
+      "loss": 2.5346,
+      "theoretical_loss": 3.364096138540153,
+      "tokens_seen": 2541486080
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.655286551919803e-05,
+      "loss": 2.5245,
+      "theoretical_loss": 3.363958247183188,
+      "tokens_seen": 2542796800
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.631643654246265e-05,
+      "loss": 2.5291,
+      "theoretical_loss": 3.3638204467760864,
+      "tokens_seen": 2544107520
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.608000756572726e-05,
+      "loss": 2.5676,
+      "theoretical_loss": 3.3636827372120495,
+      "tokens_seen": 2545418240
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5843578588991864e-05,
+      "loss": 2.5207,
+      "theoretical_loss": 3.3635451183844576,
+      "tokens_seen": 2546728960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.560714961225648e-05,
+      "loss": 2.5355,
+      "theoretical_loss": 3.3634075901868723,
+      "tokens_seen": 2548039680
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.537072063552109e-05,
+      "loss": 2.5009,
+      "theoretical_loss": 3.363270152513034,
+      "tokens_seen": 2549350400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.5134291658785704e-05,
+      "loss": 2.5245,
+      "theoretical_loss": 3.3631328052568623,
+      "tokens_seen": 2550661120
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.489786268205031e-05,
+      "loss": 2.5528,
+      "theoretical_loss": 3.3629955483124556,
+      "tokens_seen": 2551971840
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.4661433705314926e-05,
+      "loss": 2.4605,
+      "theoretical_loss": 3.3628583815740907,
+      "tokens_seen": 2553282560
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.442500472857954e-05,
+      "loss": 2.4711,
+      "theoretical_loss": 3.362721304936222,
+      "tokens_seen": 2554593280
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1399979,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.273829460144043,
+      "objective/train/theoretical_loss": 3.3625843182934827,
+      "objective/train/tokens_used": 2576364000,
+      "theoretical_loss": 3.3625843182934827,
+      "tokens_seen": 2555904000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.418857575184414e-05,
+      "loss": 2.5117,
+      "theoretical_loss": 3.3625843182934827,
+      "tokens_seen": 2555904000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.395214677510876e-05,
+      "loss": 2.4721,
+      "theoretical_loss": 3.362447421540681,
+      "tokens_seen": 2557214720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.371571779837337e-05,
+      "loss": 2.4936,
+      "theoretical_loss": 3.362310614572804,
+      "tokens_seen": 2558525440
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.347928882163798e-05,
+      "loss": 2.5827,
+      "theoretical_loss": 3.3621738972850137,
+      "tokens_seen": 2559836160
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.324285984490259e-05,
+      "loss": 2.5151,
+      "theoretical_loss": 3.3620372695726495,
+      "tokens_seen": 2561146880
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.30064308681672e-05,
+      "loss": 2.527,
+      "theoretical_loss": 3.3619007313312252,
+      "tokens_seen": 2562457600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2770001891431814e-05,
+      "loss": 2.4894,
+      "theoretical_loss": 3.36176428245643,
+      "tokens_seen": 2563768320
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.253357291469643e-05,
+      "loss": 2.4741,
+      "theoretical_loss": 3.361627922844129,
+      "tokens_seen": 2565079040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.2297143937961036e-05,
+      "loss": 2.5059,
+      "theoretical_loss": 3.3614916523903604,
+      "tokens_seen": 2566389760
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.206071496122565e-05,
+      "loss": 2.5269,
+      "theoretical_loss": 3.361355470991338,
+      "tokens_seen": 2567700480
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.182428598449026e-05,
+      "loss": 2.5196,
+      "theoretical_loss": 3.3612193785434474,
+      "tokens_seen": 2569011200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.1587857007754876e-05,
+      "loss": 2.5199,
+      "theoretical_loss": 3.3610833749432496,
+      "tokens_seen": 2570321920
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.135142803101948e-05,
+      "loss": 2.4744,
+      "theoretical_loss": 3.360947460087477,
+      "tokens_seen": 2571632640
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.111499905428409e-05,
+      "loss": 2.5587,
+      "theoretical_loss": 3.360811633873035,
+      "tokens_seen": 2572943360
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.087857007754871e-05,
+      "loss": 2.5266,
+      "theoretical_loss": 3.3606758961970016,
+      "tokens_seen": 2574254080
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.064214110081332e-05,
+      "loss": 2.5498,
+      "theoretical_loss": 3.3605402469566266,
+      "tokens_seen": 2575564800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0405712124077924e-05,
+      "loss": 2.5401,
+      "theoretical_loss": 3.3604046860493306,
+      "tokens_seen": 2576875520
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.016928314734254e-05,
+      "loss": 2.5033,
+      "theoretical_loss": 3.360269213372706,
+      "tokens_seen": 2578186240
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.993285417060715e-05,
+      "loss": 2.5329,
+      "theoretical_loss": 3.3601338288245155,
+      "tokens_seen": 2579496960
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.969642519387176e-05,
+      "loss": 2.5503,
+      "theoretical_loss": 3.3599985323026926,
+      "tokens_seen": 2580807680
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9459996217136375e-05,
+      "loss": 2.5442,
+      "theoretical_loss": 3.3598633237053397,
+      "tokens_seen": 2582118400
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.9223567240400986e-05,
+      "loss": 2.5684,
+      "theoretical_loss": 3.3597282029307305,
+      "tokens_seen": 2583429120
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.89871382636656e-05,
+      "loss": 2.5258,
+      "theoretical_loss": 3.3595931698773063,
+      "tokens_seen": 2584739840
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.87507092869302e-05,
+      "loss": 2.5794,
+      "theoretical_loss": 3.3594582244436784,
+      "tokens_seen": 2586050560
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.851428031019482e-05,
+      "loss": 2.5258,
+      "theoretical_loss": 3.3593233665286255,
+      "tokens_seen": 2587361280
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.827785133345943e-05,
+      "loss": 2.5029,
+      "theoretical_loss": 3.3591885960310957,
+      "tokens_seen": 2588672000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.804142235672404e-05,
+      "loss": 2.5185,
+      "theoretical_loss": 3.3590539128502037,
+      "tokens_seen": 2589982720
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.780499337998865e-05,
+      "loss": 2.4506,
+      "theoretical_loss": 3.358919316885233,
+      "tokens_seen": 2591293440
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.756856440325326e-05,
+      "loss": 2.4588,
+      "theoretical_loss": 3.358784808035632,
+      "tokens_seen": 2592604160
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.7332135426517874e-05,
+      "loss": 2.4594,
+      "theoretical_loss": 3.3586503862010186,
+      "tokens_seen": 2593914880
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.709570644978249e-05,
+      "loss": 2.4537,
+      "theoretical_loss": 3.3585160512811743,
+      "tokens_seen": 2595225600
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6859277473047096e-05,
+      "loss": 2.4668,
+      "theoretical_loss": 3.358381803176048,
+      "tokens_seen": 2596536320
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.662284849631171e-05,
+      "loss": 2.4729,
+      "theoretical_loss": 3.3582476417857543,
+      "tokens_seen": 2597847040
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6386419519576325e-05,
+      "loss": 2.4577,
+      "theoretical_loss": 3.3581135670105726,
+      "tokens_seen": 2599157760
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.614999054284093e-05,
+      "loss": 2.5333,
+      "theoretical_loss": 3.3579795787509474,
+      "tokens_seen": 2600468480
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.591356156610554e-05,
+      "loss": 2.4906,
+      "theoretical_loss": 3.357845676907487,
+      "tokens_seen": 2601779200
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.567713258937015e-05,
+      "loss": 2.4854,
+      "theoretical_loss": 3.357711861380966,
+      "tokens_seen": 2603089920
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.544070361263477e-05,
+      "loss": 2.4722,
+      "theoretical_loss": 3.3575781320723195,
+      "tokens_seen": 2604400640
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.5204274635899373e-05,
+      "loss": 2.4389,
+      "theoretical_loss": 3.3574444888826487,
+      "tokens_seen": 2605711360
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4967845659163984e-05,
+      "loss": 2.4752,
+      "theoretical_loss": 3.3573109317132177,
+      "tokens_seen": 2607022080
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.47314166824286e-05,
+      "loss": 2.4677,
+      "theoretical_loss": 3.3571774604654516,
+      "tokens_seen": 2608332800
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.449498770569321e-05,
+      "loss": 2.5332,
+      "theoretical_loss": 3.3570440750409407,
+      "tokens_seen": 2609643520
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.425855872895782e-05,
+      "loss": 2.4645,
+      "theoretical_loss": 3.3569107753414342,
+      "tokens_seen": 2610954240
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.4022129752222435e-05,
+      "loss": 2.5112,
+      "theoretical_loss": 3.3567775612688457,
+      "tokens_seen": 2612264960
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.3785700775487046e-05,
+      "loss": 2.5,
+      "theoretical_loss": 3.356644432725249,
+      "tokens_seen": 2613575680
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.354927179875166e-05,
+      "loss": 2.5057,
+      "theoretical_loss": 3.3565113896128786,
+      "tokens_seen": 2614886400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.331284282201626e-05,
+      "loss": 2.4667,
+      "theoretical_loss": 3.3563784318341305,
+      "tokens_seen": 2616197120
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.307641384528088e-05,
+      "loss": 2.4999,
+      "theoretical_loss": 3.3562455592915605,
+      "tokens_seen": 2617507840
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.283998486854549e-05,
+      "loss": 2.4772,
+      "theoretical_loss": 3.3561127718878843,
+      "tokens_seen": 2618818560
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2603555891810095e-05,
+      "loss": 2.4471,
+      "theoretical_loss": 3.355980069525978,
+      "tokens_seen": 2620129280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5261862578455168,
+      "debugging/distinct-1-grams": 0.7625393793155252,
+      "debugging/distinct-2-grams": 0.964892942096682,
+      "debugging/entropy-1-grams": 5.977605349866035,
+      "debugging/entropy-2-grams": 7.005058054139759,
+      "debugging/length": 540.5333333333333,
+      "debugging/num_segments": 15,
+      "epoch": 0.94,
+      "objective/train/docs_used": 1437215,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.646803379058838,
+      "objective/train/theoretical_loss": 3.3558474521088764,
+      "objective/train/tokens_used": 2641900000,
+      "theoretical_loss": 3.3558474521088764,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.236712691507471e-05,
+      "loss": 2.5258,
+      "theoretical_loss": 3.3558474521088764,
+      "tokens_seen": 2621440000
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.2130697938339324e-05,
+      "loss": 2.441,
+      "theoretical_loss": 3.355714919539773,
+      "tokens_seen": 2622750720
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1894268961603935e-05,
+      "loss": 2.5213,
+      "theoretical_loss": 3.3555824717220206,
+      "tokens_seen": 2624061440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.1657839984868546e-05,
+      "loss": 2.4339,
+      "theoretical_loss": 3.35545010855913,
+      "tokens_seen": 2625372160
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.142141100813316e-05,
+      "loss": 2.4972,
+      "theoretical_loss": 3.35531782995477,
+      "tokens_seen": 2626682880
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.118498203139777e-05,
+      "loss": 2.5092,
+      "theoretical_loss": 3.3551856358127674,
+      "tokens_seen": 2627993600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.094855305466238e-05,
+      "loss": 2.5278,
+      "theoretical_loss": 3.3550535260371053,
+      "tokens_seen": 2629304320
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.071212407792699e-05,
+      "loss": 2.5059,
+      "theoretical_loss": 3.3549215005319244,
+      "tokens_seen": 2630615040
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.04756951011916e-05,
+      "loss": 2.4837,
+      "theoretical_loss": 3.354789559201522,
+      "tokens_seen": 2631925760
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0239266124456215e-05,
+      "loss": 2.4942,
+      "theoretical_loss": 3.354657701950352,
+      "tokens_seen": 2633236480
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.0002837147720826e-05,
+      "loss": 2.4895,
+      "theoretical_loss": 3.354525928683023,
+      "tokens_seen": 2634547200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9766408170985437e-05,
+      "loss": 2.4948,
+      "theoretical_loss": 3.354394239304301,
+      "tokens_seen": 2635857920
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.9529979194250048e-05,
+      "loss": 2.4586,
+      "theoretical_loss": 3.3542626337191055,
+      "tokens_seen": 2637168640
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.929355021751466e-05,
+      "loss": 2.4407,
+      "theoretical_loss": 3.3541311118325123,
+      "tokens_seen": 2638479360
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.905712124077927e-05,
+      "loss": 2.4761,
+      "theoretical_loss": 3.3539996735497515,
+      "tokens_seen": 2639790080
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8820692264043885e-05,
+      "loss": 2.4699,
+      "theoretical_loss": 3.3538683187762066,
+      "tokens_seen": 2641100800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8584263287308492e-05,
+      "loss": 2.4686,
+      "theoretical_loss": 3.3537370474174164,
+      "tokens_seen": 2642411520
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8347834310573103e-05,
+      "loss": 2.505,
+      "theoretical_loss": 3.3536058593790727,
+      "tokens_seen": 2643722240
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.8111405333837714e-05,
+      "loss": 2.5144,
+      "theoretical_loss": 3.3534747545670207,
+      "tokens_seen": 2645032960
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7874976357102325e-05,
+      "loss": 2.5098,
+      "theoretical_loss": 3.3533437328872577,
+      "tokens_seen": 2646343680
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.763854738036694e-05,
+      "loss": 2.5305,
+      "theoretical_loss": 3.3532127942459353,
+      "tokens_seen": 2647654400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7402118403631547e-05,
+      "loss": 2.4925,
+      "theoretical_loss": 3.3530819385493564,
+      "tokens_seen": 2648965120
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.7165689426896162e-05,
+      "loss": 2.5039,
+      "theoretical_loss": 3.3529511657039763,
+      "tokens_seen": 2650275840
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6929260450160773e-05,
+      "loss": 2.4693,
+      "theoretical_loss": 3.3528204756164017,
+      "tokens_seen": 2651586560
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6692831473425384e-05,
+      "loss": 2.4967,
+      "theoretical_loss": 3.3526898681933908,
+      "tokens_seen": 2652897280
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6456402496689995e-05,
+      "loss": 2.5326,
+      "theoretical_loss": 3.3525593433418526,
+      "tokens_seen": 2654208000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.6219973519954606e-05,
+      "loss": 2.4533,
+      "theoretical_loss": 3.3524289009688473,
+      "tokens_seen": 2655518720
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5983544543219217e-05,
+      "loss": 2.4789,
+      "theoretical_loss": 3.352298540981586,
+      "tokens_seen": 2656829440
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.574711556648383e-05,
+      "loss": 2.514,
+      "theoretical_loss": 3.352168263287428,
+      "tokens_seen": 2658140160
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.551068658974844e-05,
+      "loss": 2.5284,
+      "theoretical_loss": 3.3520380677938846,
+      "tokens_seen": 2659450880
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.5274257613013053e-05,
+      "loss": 2.4977,
+      "theoretical_loss": 3.351907954408615,
+      "tokens_seen": 2660761600
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.503782863627766e-05,
+      "loss": 2.48,
+      "theoretical_loss": 3.3517779230394282,
+      "tokens_seen": 2662072320
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4801399659542272e-05,
+      "loss": 2.4684,
+      "theoretical_loss": 3.3516479735942823,
+      "tokens_seen": 2663383040
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4564970682806887e-05,
+      "loss": 2.553,
+      "theoretical_loss": 3.3515181059812837,
+      "tokens_seen": 2664693760
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.4328541706071494e-05,
+      "loss": 2.5593,
+      "theoretical_loss": 3.3513883201086863,
+      "tokens_seen": 2666004480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.409211272933611e-05,
+      "loss": 2.5372,
+      "theoretical_loss": 3.351258615884893,
+      "tokens_seen": 2667315200
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.385568375260072e-05,
+      "loss": 2.5743,
+      "theoretical_loss": 3.3511289932184534,
+      "tokens_seen": 2668625920
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.361925477586533e-05,
+      "loss": 2.5508,
+      "theoretical_loss": 3.3509994520180655,
+      "tokens_seen": 2669936640
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3382825799129942e-05,
+      "loss": 2.5263,
+      "theoretical_loss": 3.3508699921925724,
+      "tokens_seen": 2671247360
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.3146396822394553e-05,
+      "loss": 2.5608,
+      "theoretical_loss": 3.350740613650966,
+      "tokens_seen": 2672558080
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.2909967845659164e-05,
+      "loss": 2.6359,
+      "theoretical_loss": 3.350611316302383,
+      "tokens_seen": 2673868800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2673538868923778e-05,
+      "loss": 2.5421,
+      "theoretical_loss": 3.3504821000561074,
+      "tokens_seen": 2675179520
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2437109892188386e-05,
+      "loss": 2.5302,
+      "theoretical_loss": 3.3503529648215675,
+      "tokens_seen": 2676490240
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.2200680915453e-05,
+      "loss": 2.5156,
+      "theoretical_loss": 3.350223910508338,
+      "tokens_seen": 2677800960
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1964251938717608e-05,
+      "loss": 2.5685,
+      "theoretical_loss": 3.350094937026139,
+      "tokens_seen": 2679111680
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1727822961982222e-05,
+      "loss": 2.5245,
+      "theoretical_loss": 3.349966044284834,
+      "tokens_seen": 2680422400
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1491393985246833e-05,
+      "loss": 2.6009,
+      "theoretical_loss": 3.3498372321944325,
+      "tokens_seen": 2681733120
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1254965008511444e-05,
+      "loss": 2.5898,
+      "theoretical_loss": 3.349708500665088,
+      "tokens_seen": 2683043840
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1018536031776055e-05,
+      "loss": 2.5696,
+      "theoretical_loss": 3.3495798496070974,
+      "tokens_seen": 2684354560
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0782107055040663e-05,
+      "loss": 2.6125,
+      "theoretical_loss": 3.3494512789309008,
+      "tokens_seen": 2685665280
+    },
+    {
+      "epoch": 0.96,
+      "objective/train/docs_used": 1472652,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.3159241676330566,
+      "objective/train/theoretical_loss": 3.3493227885470835,
+      "objective/train/tokens_used": 2707436000,
+      "theoretical_loss": 3.3493227885470835,
+      "tokens_seen": 2686976000
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.0545678078305277e-05,
+      "loss": 2.5558,
+      "theoretical_loss": 3.3493227885470835,
+      "tokens_seen": 2686976000
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.030924910156989e-05,
+      "loss": 2.593,
+      "theoretical_loss": 3.349194378366372,
+      "tokens_seen": 2688286720
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.00728201248345e-05,
+      "loss": 2.5223,
+      "theoretical_loss": 3.349066048299635,
+      "tokens_seen": 2689597440
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.983639114809911e-05,
+      "loss": 2.5658,
+      "theoretical_loss": 3.3489377982578863,
+      "tokens_seen": 2690908160
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9599962171363725e-05,
+      "loss": 2.5847,
+      "theoretical_loss": 3.3488096281522797,
+      "tokens_seen": 2692218880
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9363533194628333e-05,
+      "loss": 2.5862,
+      "theoretical_loss": 3.3486815378941115,
+      "tokens_seen": 2693529600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9127104217892947e-05,
+      "loss": 2.5543,
+      "theoretical_loss": 3.348553527394819,
+      "tokens_seen": 2694840320
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.8890675241157555e-05,
+      "loss": 2.5645,
+      "theoretical_loss": 3.3484255965659813,
+      "tokens_seen": 2696151040
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.865424626442217e-05,
+      "loss": 2.5306,
+      "theoretical_loss": 3.348297745319319,
+      "tokens_seen": 2697461760
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.841781728768678e-05,
+      "loss": 2.597,
+      "theoretical_loss": 3.348169973566691,
+      "tokens_seen": 2698772480
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.818138831095139e-05,
+      "loss": 2.5784,
+      "theoretical_loss": 3.3480422812201,
+      "tokens_seen": 2700083200
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.7944959334216002e-05,
+      "loss": 2.5689,
+      "theoretical_loss": 3.3479146681916863,
+      "tokens_seen": 2701393920
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7708530357480613e-05,
+      "loss": 2.541,
+      "theoretical_loss": 3.3477871343937307,
+      "tokens_seen": 2702704640
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7472101380745224e-05,
+      "loss": 2.5599,
+      "theoretical_loss": 3.3476596797386535,
+      "tokens_seen": 2704015360
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7235672404009835e-05,
+      "loss": 2.5374,
+      "theoretical_loss": 3.347532304139014,
+      "tokens_seen": 2705326080
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6999243427274446e-05,
+      "loss": 2.555,
+      "theoretical_loss": 3.347405007507511,
+      "tokens_seen": 2706636800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6762814450539057e-05,
+      "loss": 2.4958,
+      "theoretical_loss": 3.3472777897569808,
+      "tokens_seen": 2707947520
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.652638547380367e-05,
+      "loss": 2.4926,
+      "theoretical_loss": 3.3471506508003994,
+      "tokens_seen": 2709258240
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.628995649706828e-05,
+      "loss": 2.5153,
+      "theoretical_loss": 3.3470235905508803,
+      "tokens_seen": 2710568960
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6053527520332894e-05,
+      "loss": 2.593,
+      "theoretical_loss": 3.346896608921674,
+      "tokens_seen": 2711879680
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.58170985435975e-05,
+      "loss": 2.5494,
+      "theoretical_loss": 3.346769705826169,
+      "tokens_seen": 2713190400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5580669566862116e-05,
+      "loss": 2.5465,
+      "theoretical_loss": 3.3466428811778925,
+      "tokens_seen": 2714501120
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5344240590126727e-05,
+      "loss": 2.5772,
+      "theoretical_loss": 3.346516134890506,
+      "tokens_seen": 2715811840
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5107811613391336e-05,
+      "loss": 2.5505,
+      "theoretical_loss": 3.3463894668778096,
+      "tokens_seen": 2717122560
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4871382636655949e-05,
+      "loss": 2.5324,
+      "theoretical_loss": 3.3462628770537384,
+      "tokens_seen": 2718433280
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.463495365992056e-05,
+      "loss": 2.5461,
+      "theoretical_loss": 3.346136365332365,
+      "tokens_seen": 2719744000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4398524683185171e-05,
+      "loss": 2.5398,
+      "theoretical_loss": 3.3460099316278957,
+      "tokens_seen": 2721054720
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.4162095706449782e-05,
+      "loss": 2.5144,
+      "theoretical_loss": 3.3458835758546748,
+      "tokens_seen": 2722365440
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3925666729714395e-05,
+      "loss": 2.542,
+      "theoretical_loss": 3.34575729792718,
+      "tokens_seen": 2723676160
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3689237752979006e-05,
+      "loss": 2.5032,
+      "theoretical_loss": 3.345631097760025,
+      "tokens_seen": 2724986880
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3452808776243617e-05,
+      "loss": 2.4663,
+      "theoretical_loss": 3.3455049752679575,
+      "tokens_seen": 2726297600
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.3216379799508228e-05,
+      "loss": 2.5059,
+      "theoretical_loss": 3.34537893036586,
+      "tokens_seen": 2727608320
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2979950822772839e-05,
+      "loss": 2.6391,
+      "theoretical_loss": 3.345252962968749,
+      "tokens_seen": 2728919040
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2743521846037451e-05,
+      "loss": 2.5071,
+      "theoretical_loss": 3.345127072991774,
+      "tokens_seen": 2730229760
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2507092869302062e-05,
+      "loss": 2.527,
+      "theoretical_loss": 3.3450012603502204,
+      "tokens_seen": 2731540480
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2270663892566673e-05,
+      "loss": 2.5683,
+      "theoretical_loss": 3.3448755249595044,
+      "tokens_seen": 2732851200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.2034234915831285e-05,
+      "loss": 2.5626,
+      "theoretical_loss": 3.3447498667351754,
+      "tokens_seen": 2734161920
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1797805939095897e-05,
+      "loss": 2.5557,
+      "theoretical_loss": 3.344624285592918,
+      "tokens_seen": 2735472640
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1561376962360508e-05,
+      "loss": 2.5683,
+      "theoretical_loss": 3.3444987814485465,
+      "tokens_seen": 2736783360
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1324947985625118e-05,
+      "loss": 2.5486,
+      "theoretical_loss": 3.3443733542180087,
+      "tokens_seen": 2738094080
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1088519008889729e-05,
+      "loss": 2.5569,
+      "theoretical_loss": 3.344248003817384,
+      "tokens_seen": 2739404800
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0852090032154341e-05,
+      "loss": 2.5722,
+      "theoretical_loss": 3.3441227301628844,
+      "tokens_seen": 2740715520
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0615661055418952e-05,
+      "loss": 2.5424,
+      "theoretical_loss": 3.343997533170852,
+      "tokens_seen": 2742026240
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0379232078683563e-05,
+      "loss": 2.563,
+      "theoretical_loss": 3.34387241275776,
+      "tokens_seen": 2743336960
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0142803101948174e-05,
+      "loss": 2.5885,
+      "theoretical_loss": 3.3437473688402135,
+      "tokens_seen": 2744647680
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.906374125212785e-06,
+      "loss": 2.6019,
+      "theoretical_loss": 3.343622401334948,
+      "tokens_seen": 2745958400
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.669945148477398e-06,
+      "loss": 2.5506,
+      "theoretical_loss": 3.3434975101588282,
+      "tokens_seen": 2747269120
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.43351617174201e-06,
+      "loss": 2.5464,
+      "theoretical_loss": 3.34337269522885,
+      "tokens_seen": 2748579840
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 9.19708719500662e-06,
+      "loss": 2.5645,
+      "theoretical_loss": 3.343247956462139,
+      "tokens_seen": 2749890560
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.960658218271231e-06,
+      "loss": 2.5346,
+      "theoretical_loss": 3.3431232937759496,
+      "tokens_seen": 2751201280
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1508642,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7461962699890137,
+      "objective/train/theoretical_loss": 3.3429987070876668,
+      "objective/train/tokens_used": 2772972000,
+      "theoretical_loss": 3.3429987070876668,
+      "tokens_seen": 2752512000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.724229241535844e-06,
+      "loss": 2.5547,
+      "theoretical_loss": 3.3429987070876668,
+      "tokens_seen": 2752512000
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.487800264800455e-06,
+      "loss": 2.5661,
+      "theoretical_loss": 3.3428741963148028,
+      "tokens_seen": 2753822720
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.251371288065066e-06,
+      "loss": 2.5112,
+      "theoretical_loss": 3.3427497613749995,
+      "tokens_seen": 2755133440
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.014942311329677e-06,
+      "loss": 2.5537,
+      "theoretical_loss": 3.3426254021860276,
+      "tokens_seen": 2756444160
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.778513334594288e-06,
+      "loss": 2.5529,
+      "theoretical_loss": 3.342501118665785,
+      "tokens_seen": 2757754880
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.542084357858899e-06,
+      "loss": 2.5028,
+      "theoretical_loss": 3.3423769107322996,
+      "tokens_seen": 2759065600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.305655381123511e-06,
+      "loss": 2.5189,
+      "theoretical_loss": 3.342252778303724,
+      "tokens_seen": 2760376320
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.069226404388122e-06,
+      "loss": 2.4983,
+      "theoretical_loss": 3.342128721298341,
+      "tokens_seen": 2761687040
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.832797427652734e-06,
+      "loss": 2.4668,
+      "theoretical_loss": 3.342004739634559,
+      "tokens_seen": 2762997760
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.596368450917344e-06,
+      "loss": 2.5218,
+      "theoretical_loss": 3.3418808332309133,
+      "tokens_seen": 2764308480
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.359939474181956e-06,
+      "loss": 2.4873,
+      "theoretical_loss": 3.3417570020060667,
+      "tokens_seen": 2765619200
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.123510497446567e-06,
+      "loss": 2.5271,
+      "theoretical_loss": 3.341633245878808,
+      "tokens_seen": 2766929920
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.887081520711179e-06,
+      "loss": 2.5086,
+      "theoretical_loss": 3.3415095647680517,
+      "tokens_seen": 2768240640
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.65065254397579e-06,
+      "loss": 2.4794,
+      "theoretical_loss": 3.341385958592839,
+      "tokens_seen": 2769551360
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.414223567240401e-06,
+      "loss": 2.5135,
+      "theoretical_loss": 3.3412624272723366,
+      "tokens_seen": 2770862080
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.177794590505013e-06,
+      "loss": 2.5404,
+      "theoretical_loss": 3.3411389707258357,
+      "tokens_seen": 2772172800
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.941365613769624e-06,
+      "loss": 2.4904,
+      "theoretical_loss": 3.3410155888727537,
+      "tokens_seen": 2773483520
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.704936637034235e-06,
+      "loss": 2.5394,
+      "theoretical_loss": 3.340892281632632,
+      "tokens_seen": 2774794240
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.468507660298846e-06,
+      "loss": 2.5343,
+      "theoretical_loss": 3.340769048925137,
+      "tokens_seen": 2776104960
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.232078683563458e-06,
+      "loss": 2.5252,
+      "theoretical_loss": 3.3406458906700602,
+      "tokens_seen": 2777415680
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.995649706828069e-06,
+      "loss": 2.5006,
+      "theoretical_loss": 3.340522806787316,
+      "tokens_seen": 2778726400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.75922073009268e-06,
+      "loss": 2.5205,
+      "theoretical_loss": 3.3403997971969432,
+      "tokens_seen": 2780037120
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.5227917533572916e-06,
+      "loss": 2.4855,
+      "theoretical_loss": 3.3402768618191043,
+      "tokens_seen": 2781347840
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2863627766219027e-06,
+      "loss": 2.54,
+      "theoretical_loss": 3.3401540005740857,
+      "tokens_seen": 2782658560
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.049933799886514e-06,
+      "loss": 2.5011,
+      "theoretical_loss": 3.340031213382295,
+      "tokens_seen": 2783969280
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.8135048231511256e-06,
+      "loss": 2.474,
+      "theoretical_loss": 3.339908500164265,
+      "tokens_seen": 2785280000
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.577075846415737e-06,
+      "loss": 2.535,
+      "theoretical_loss": 3.3397858608406494,
+      "tokens_seen": 2786590720
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.340646869680348e-06,
+      "loss": 2.5224,
+      "theoretical_loss": 3.339663295332226,
+      "tokens_seen": 2787901440
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.1042178929449595e-06,
+      "loss": 2.5139,
+      "theoretical_loss": 3.339540803559893,
+      "tokens_seen": 2789212160
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.8677889162095707e-06,
+      "loss": 2.5674,
+      "theoretical_loss": 3.3394183854446706,
+      "tokens_seen": 2790522880
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.631359939474182e-06,
+      "loss": 2.5862,
+      "theoretical_loss": 3.339296040907702,
+      "tokens_seen": 2791833600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.3949309627387934e-06,
+      "loss": 2.5995,
+      "theoretical_loss": 3.3391737698702517,
+      "tokens_seen": 2793144320
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.1585019860034044e-06,
+      "loss": 2.5867,
+      "theoretical_loss": 3.339051572253703,
+      "tokens_seen": 2794455040
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.220730092680159e-07,
+      "loss": 2.6028,
+      "theoretical_loss": 3.3389294479795635,
+      "tokens_seen": 2795765760
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.856440325326272e-07,
+      "loss": 2.6548,
+      "theoretical_loss": 3.3388073969694583,
+      "tokens_seen": 2797076480
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.4921505579723853e-07,
+      "loss": 2.5601,
+      "theoretical_loss": 3.338685419145136,
+      "tokens_seen": 2798387200
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.1278607906184985e-07,
+      "loss": 2.6531,
+      "theoretical_loss": 3.3385635144284618,
+      "tokens_seen": 2799697920
+    }
+  ],
+  "max_steps": 42724,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 1.428920419221504e+18,
+  "trial_name": null,
+  "trial_params": null
+}