diff --git "a/checkpoint-21362/trainer_state.json" "b/checkpoint-21362/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-21362/trainer_state.json"
@@ -0,0 +1,7758 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5,
+  "global_step": 21362,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1682242990654204e-06,
+      "loss": 10.876,
+      "theoretical_loss": 20.812802717066116,
+      "tokens_seen": 65536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.336448598130841e-05,
+      "loss": 9.7133,
+      "theoretical_loss": 10.363548859282295,
+      "tokens_seen": 1310720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.672897196261682e-05,
+      "loss": 8.8628,
+      "theoretical_loss": 8.96027440542832,
+      "tokens_seen": 2621440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.009345794392523e-05,
+      "loss": 7.9844,
+      "theoretical_loss": 8.257160837770828,
+      "tokens_seen": 3932160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.345794392523364e-05,
+      "loss": 7.1152,
+      "theoretical_loss": 7.80455017047313,
+      "tokens_seen": 5242880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011682242990654206,
+      "loss": 6.5777,
+      "theoretical_loss": 7.477743633566995,
+      "tokens_seen": 6553600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014018691588785047,
+      "loss": 6.3198,
+      "theoretical_loss": 7.225472152031461,
+      "tokens_seen": 7864320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016355140186915886,
+      "loss": 6.0818,
+      "theoretical_loss": 7.022000360175642,
+      "tokens_seen": 9175040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018691588785046728,
+      "loss": 5.9629,
+      "theoretical_loss": 6.852706072125594,
+      "tokens_seen": 10485760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002102803738317757,
+      "loss": 5.7999,
+      "theoretical_loss": 6.7085424020813775,
+      "tokens_seen": 11796480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023364485981308412,
+      "loss": 5.6915,
+      "theoretical_loss": 6.583551143945378,
+      "tokens_seen": 13107200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0002570093457943925,
+      "loss": 5.592,
+      "theoretical_loss": 6.4736158466608344,
+      "tokens_seen": 14417920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028037383177570094,
+      "loss": 5.499,
+      "theoretical_loss": 6.375782617839931,
+      "tokens_seen": 15728640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00030373831775700936,
+      "loss": 5.3849,
+      "theoretical_loss": 6.287865514264695,
+      "tokens_seen": 17039360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003271028037383177,
+      "loss": 5.3163,
+      "theoretical_loss": 6.208205077800142,
+      "tokens_seen": 18350080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035046728971962614,
+      "loss": 5.2568,
+      "theoretical_loss": 6.13551414745929,
+      "tokens_seen": 19660800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037383177570093456,
+      "loss": 5.2262,
+      "theoretical_loss": 6.068775822887412,
+      "tokens_seen": 20971520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000397196261682243,
+      "loss": 5.1178,
+      "theoretical_loss": 6.007173857995872,
+      "tokens_seen": 22282240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004205607476635514,
+      "loss": 5.0643,
+      "theoretical_loss": 5.950043919222669,
+      "tokens_seen": 23592960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004439252336448598,
+      "loss": 5.0518,
+      "theoretical_loss": 5.89683866811525,
+      "tokens_seen": 24903680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046728971962616824,
+      "loss": 5.0122,
+      "theoretical_loss": 5.847102241784889,
+      "tokens_seen": 26214400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004906542056074767,
+      "loss": 4.8736,
+      "theoretical_loss": 5.80045126829951,
+      "tokens_seen": 27525120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998581426139588,
+      "loss": 4.9113,
+      "theoretical_loss": 5.756560518460697,
+      "tokens_seen": 28835840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996217136372233,
+      "loss": 4.8605,
+      "theoretical_loss": 5.715151906614886,
+      "tokens_seen": 30146560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499385284660488,
+      "loss": 4.7564,
+      "theoretical_loss": 5.675985950013518,
+      "tokens_seen": 31457280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991488556837526,
+      "loss": 4.7565,
+      "theoretical_loss": 5.638855059590318,
+      "tokens_seen": 32768000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989124267070172,
+      "loss": 4.6969,
+      "theoretical_loss": 5.603578213244772,
+      "tokens_seen": 34078720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986759977302818,
+      "loss": 4.6145,
+      "theoretical_loss": 5.569996685496657,
+      "tokens_seen": 35389440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984395687535465,
+      "loss": 4.6134,
+      "theoretical_loss": 5.53797059334095,
+      "tokens_seen": 36700160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498203139776811,
+      "loss": 4.587,
+      "theoretical_loss": 5.507376079224074,
+      "tokens_seen": 38010880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979667108000757,
+      "loss": 4.5379,
+      "theoretical_loss": 5.478102996075402,
+      "tokens_seen": 39321600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977302818233403,
+      "loss": 4.5217,
+      "theoretical_loss": 5.450052991438716,
+      "tokens_seen": 40632320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974938528466049,
+      "loss": 4.4559,
+      "theoretical_loss": 5.4231379114505165,
+      "tokens_seen": 41943040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972574238698695,
+      "loss": 4.4216,
+      "theoretical_loss": 5.397278463098006,
+      "tokens_seen": 43253760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970209948931342,
+      "loss": 4.5159,
+      "theoretical_loss": 5.372403086519811,
+      "tokens_seen": 44564480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967845659163987,
+      "loss": 4.365,
+      "theoretical_loss": 5.348446999254799,
+      "tokens_seen": 45875200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965481369396634,
+      "loss": 4.3884,
+      "theoretical_loss": 5.3253513821291545,
+      "tokens_seen": 47185920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496311707962928,
+      "loss": 4.3143,
+      "theoretical_loss": 5.303062682496617,
+      "tokens_seen": 48496640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960752789861926,
+      "loss": 4.3191,
+      "theoretical_loss": 5.281532015245618,
+      "tokens_seen": 49807360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958388500094572,
+      "loss": 4.2536,
+      "theoretical_loss": 5.260714645678606,
+      "tokens_seen": 51118080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956024210327218,
+      "loss": 4.264,
+      "theoretical_loss": 5.240569541288942,
+      "tokens_seen": 52428800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953659920559864,
+      "loss": 4.2381,
+      "theoretical_loss": 5.221058981785759,
+      "tokens_seen": 53739520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495129563079251,
+      "loss": 4.2405,
+      "theoretical_loss": 5.202148218579597,
+      "tokens_seen": 55050240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948931341025156,
+      "loss": 4.1774,
+      "theoretical_loss": 5.1838051764423945,
+      "tokens_seen": 56360960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946567051257802,
+      "loss": 4.1062,
+      "theoretical_loss": 5.166000191271213,
+      "tokens_seen": 57671680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944202761490448,
+      "loss": 4.0461,
+      "theoretical_loss": 5.148705778875405,
+      "tokens_seen": 58982400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941838471723094,
+      "loss": 4.0506,
+      "theoretical_loss": 5.131896430517482,
+      "tokens_seen": 60293120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493947418195574,
+      "loss": 3.9962,
+      "theoretical_loss": 5.1155484316045445,
+      "tokens_seen": 61603840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937109892188386,
+      "loss": 3.9316,
+      "theoretical_loss": 5.099639700477905,
+      "tokens_seen": 62914560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934745602421033,
+      "loss": 3.9085,
+      "theoretical_loss": 5.084149644705477,
+      "tokens_seen": 64225280
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 46774,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.699275493621826,
+      "objective/train/theoretical_loss": 5.06905903266221,
+      "objective/train/tokens_used": 85996000,
+      "theoretical_loss": 5.06905903266221,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932381312653678,
+      "loss": 3.8543,
+      "theoretical_loss": 5.06905903266221,
+      "tokens_seen": 65536000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930017022886325,
+      "loss": 3.8104,
+      "theoretical_loss": 5.054349878502332,
+      "tokens_seen": 66846720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927652733118971,
+      "loss": 3.8484,
+      "theoretical_loss": 5.04000533889449,
+      "tokens_seen": 68157440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925288443351617,
+      "loss": 3.8394,
+      "theoretical_loss": 5.02600962011627,
+      "tokens_seen": 69468160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922924153584263,
+      "loss": 3.7804,
+      "theoretical_loss": 5.012347894295022,
+      "tokens_seen": 70778880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492055986381691,
+      "loss": 3.7392,
+      "theoretical_loss": 4.999006223743629,
+      "tokens_seen": 72089600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918195574049555,
+      "loss": 3.7454,
+      "theoretical_loss": 4.985971492477399,
+      "tokens_seen": 73400320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915831284282202,
+      "loss": 3.7814,
+      "theoretical_loss": 4.973231344115766,
+      "tokens_seen": 74711040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913466994514848,
+      "loss": 3.7235,
+      "theoretical_loss": 4.960774125473073,
+      "tokens_seen": 76021760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911102704747494,
+      "loss": 3.5992,
+      "theoretical_loss": 4.948588835229052,
+      "tokens_seen": 77332480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490873841498014,
+      "loss": 3.7035,
+      "theoretical_loss": 4.9366650771441005,
+      "tokens_seen": 78643200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906374125212787,
+      "loss": 3.6193,
+      "theoretical_loss": 4.924993017348572,
+      "tokens_seen": 79953920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904009835445432,
+      "loss": 3.6246,
+      "theoretical_loss": 4.913563345291011,
+      "tokens_seen": 81264640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901645545678079,
+      "loss": 3.6119,
+      "theoretical_loss": 4.90236723797842,
+      "tokens_seen": 82575360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899281255910725,
+      "loss": 3.6384,
+      "theoretical_loss": 4.891396327183692,
+      "tokens_seen": 83886080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896916966143371,
+      "loss": 3.5648,
+      "theoretical_loss": 4.880642669331891,
+      "tokens_seen": 85196800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894552676376017,
+      "loss": 3.5957,
+      "theoretical_loss": 4.8700987178090696,
+      "tokens_seen": 86507520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892188386608663,
+      "loss": 3.5492,
+      "theoretical_loss": 4.859757297465288,
+      "tokens_seen": 87818240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889824096841309,
+      "loss": 3.4901,
+      "theoretical_loss": 4.84961158110812,
+      "tokens_seen": 89128960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887459807073955,
+      "loss": 3.5444,
+      "theoretical_loss": 4.839655067804491,
+      "tokens_seen": 90439680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885095517306601,
+      "loss": 3.4675,
+      "theoretical_loss": 4.829881562827778,
+      "tokens_seen": 91750400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882731227539247,
+      "loss": 3.4676,
+      "theoretical_loss": 4.820285159103877,
+      "tokens_seen": 93061120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048803669377718936,
+      "loss": 3.4724,
+      "theoretical_loss": 4.810860220024846,
+      "tokens_seen": 94371840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048780026480045396,
+      "loss": 3.4715,
+      "theoretical_loss": 4.801601363511853,
+      "tokens_seen": 95682560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048756383582371855,
+      "loss": 3.4911,
+      "theoretical_loss": 4.792503447220913,
+      "tokens_seen": 96993280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873274068469832,
+      "loss": 3.5129,
+      "theoretical_loss": 4.783561554795222,
+      "tokens_seen": 98304000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870909778702478,
+      "loss": 3.4716,
+      "theoretical_loss": 4.774770983077213,
+      "tokens_seen": 99614720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868545488935124,
+      "loss": 3.4971,
+      "theoretical_loss": 4.766127230201681,
+      "tokens_seen": 100925440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486618119916777,
+      "loss": 3.4911,
+      "theoretical_loss": 4.757625984498683,
+      "tokens_seen": 102236160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048638169094004165,
+      "loss": 3.538,
+      "theoretical_loss": 4.7492631141415815,
+      "tokens_seen": 103546880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861452619633062,
+      "loss": 3.4494,
+      "theoretical_loss": 4.741034657481437,
+      "tokens_seen": 104857600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048590883298657084,
+      "loss": 3.4252,
+      "theoretical_loss": 4.732936814014305,
+      "tokens_seen": 106168320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856724040098355,
+      "loss": 3.4725,
+      "theoretical_loss": 4.724965935932751,
+      "tokens_seen": 107479040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048543597503310004,
+      "loss": 3.4667,
+      "theoretical_loss": 4.717118520217186,
+      "tokens_seen": 108789760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851995460563647,
+      "loss": 3.4787,
+      "theoretical_loss": 4.709391201226483,
+      "tokens_seen": 110100480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849631170796293,
+      "loss": 3.4269,
+      "theoretical_loss": 4.70178074375083,
+      "tokens_seen": 111411200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847266881028939,
+      "loss": 3.4638,
+      "theoretical_loss": 4.694284036492922,
+      "tokens_seen": 112721920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048449025912615854,
+      "loss": 3.4204,
+      "theoretical_loss": 4.686898085946441,
+      "tokens_seen": 114032640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048425383014942313,
+      "loss": 3.4832,
+      "theoretical_loss": 4.6796200106433545,
+      "tokens_seen": 115343360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048401740117268773,
+      "loss": 3.4661,
+      "theoretical_loss": 4.6724470357438985,
+      "tokens_seen": 116654080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048378097219595233,
+      "loss": 3.4106,
+      "theoretical_loss": 4.665376487945246,
+      "tokens_seen": 117964800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483544543219217,
+      "loss": 3.4116,
+      "theoretical_loss": 4.658405790686763,
+      "tokens_seen": 119275520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833081142424816,
+      "loss": 3.4164,
+      "theoretical_loss": 4.6515324596315395,
+      "tokens_seen": 120586240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830716852657462,
+      "loss": 3.3982,
+      "theoretical_loss": 4.6447540984054605,
+      "tokens_seen": 121896960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048283525628901083,
+      "loss": 3.4399,
+      "theoretical_loss": 4.638068394576544,
+      "tokens_seen": 123207680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048259882731227537,
+      "loss": 3.397,
+      "theoretical_loss": 4.631473115858608,
+      "tokens_seen": 124518400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048236239833554,
+      "loss": 3.4011,
+      "theoretical_loss": 4.624966106524518,
+      "tokens_seen": 125829120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821259693588047,
+      "loss": 3.429,
+      "theoretical_loss": 4.61854528401543,
+      "tokens_seen": 127139840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818895403820692,
+      "loss": 3.4388,
+      "theoretical_loss": 4.6122086357333725,
+      "tokens_seen": 128450560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048165311140533387,
+      "loss": 3.3778,
+      "theoretical_loss": 4.605954216005544,
+      "tokens_seen": 129761280
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 82490,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.494258165359497,
+      "objective/train/theoretical_loss": 4.599780143209469,
+      "objective/train/tokens_used": 151532000,
+      "theoretical_loss": 4.599780143209469,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048141668242859847,
+      "loss": 3.3488,
+      "theoretical_loss": 4.599780143209469,
+      "tokens_seen": 131072000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048118025345186306,
+      "loss": 3.3833,
+      "theoretical_loss": 4.593684597048995,
+      "tokens_seen": 132382720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809438244751277,
+      "loss": 3.4206,
+      "theoretical_loss": 4.587665815971789,
+      "tokens_seen": 133693440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807073954983923,
+      "loss": 3.385,
+      "theoretical_loss": 4.581722094719703,
+      "tokens_seen": 135004160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804709665216569,
+      "loss": 3.3348,
+      "theoretical_loss": 4.575851782003932,
+      "tokens_seen": 136314880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802345375449215,
+      "loss": 3.374,
+      "theoretical_loss": 4.570053278297481,
+      "tokens_seen": 137625600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047999810856818616,
+      "loss": 3.3027,
+      "theoretical_loss": 4.564325033737986,
+      "tokens_seen": 138936320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797616795914507,
+      "loss": 3.3032,
+      "theoretical_loss": 4.558665546134362,
+      "tokens_seen": 140247040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047952525061471536,
+      "loss": 3.3335,
+      "theoretical_loss": 4.5530733590712344,
+      "tokens_seen": 141557760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047928882163798,
+      "loss": 3.3355,
+      "theoretical_loss": 4.547547060105504,
+      "tokens_seen": 142868480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905239266124455,
+      "loss": 3.3308,
+      "theoretical_loss": 4.542085279049763,
+      "tokens_seen": 144179200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788159636845092,
+      "loss": 3.3303,
+      "theoretical_loss": 4.53668668633763,
+      "tokens_seen": 145489920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047857953470777375,
+      "loss": 3.3178,
+      "theoretical_loss": 4.531349991466399,
+      "tokens_seen": 146800640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783431057310384,
+      "loss": 3.26,
+      "theoretical_loss": 4.526073941512697,
+      "tokens_seen": 148111360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047810667675430305,
+      "loss": 3.315,
+      "theoretical_loss": 4.520857319717103,
+      "tokens_seen": 149422080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778702477775676,
+      "loss": 3.266,
+      "theoretical_loss": 4.515698944133961,
+      "tokens_seen": 150732800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047763381880083224,
+      "loss": 3.3234,
+      "theoretical_loss": 4.510597666342842,
+      "tokens_seen": 152043520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047739738982409684,
+      "loss": 3.2683,
+      "theoretical_loss": 4.5055523702183375,
+      "tokens_seen": 153354240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047716096084736144,
+      "loss": 3.2506,
+      "theoretical_loss": 4.500561970755069,
+      "tokens_seen": 154664960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769245318706261,
+      "loss": 3.2891,
+      "theoretical_loss": 4.495625412944987,
+      "tokens_seen": 155975680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766881028938907,
+      "loss": 3.2946,
+      "theoretical_loss": 4.490741670704231,
+      "tokens_seen": 157286400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764516739171553,
+      "loss": 3.2928,
+      "theoretical_loss": 4.485909745846939,
+      "tokens_seen": 158597120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762152449404199,
+      "loss": 3.268,
+      "theoretical_loss": 4.481128667103624,
+      "tokens_seen": 159907840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047597881596368453,
+      "loss": 3.2522,
+      "theoretical_loss": 4.476397489181794,
+      "tokens_seen": 161218560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047574238698694913,
+      "loss": 3.2549,
+      "theoretical_loss": 4.471715291866703,
+      "tokens_seen": 162529280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047550595801021373,
+      "loss": 3.2548,
+      "theoretical_loss": 4.467081179160184,
+      "tokens_seen": 163840000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752695290334784,
+      "loss": 3.2945,
+      "theoretical_loss": 4.462494278455688,
+      "tokens_seen": 165150720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750331000567429,
+      "loss": 3.2251,
+      "theoretical_loss": 4.457953739747708,
+      "tokens_seen": 166461440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747966710800076,
+      "loss": 3.243,
+      "theoretical_loss": 4.453458734873907,
+      "tokens_seen": 167772160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745602421032722,
+      "loss": 3.1643,
+      "theoretical_loss": 4.449008456788359,
+      "tokens_seen": 169082880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432381312653677,
+      "loss": 3.2359,
+      "theoretical_loss": 4.4446021188643945,
+      "tokens_seen": 170393600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740873841498014,
+      "loss": 3.2465,
+      "theoretical_loss": 4.440238954225603,
+      "tokens_seen": 171704320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473850955173066,
+      "loss": 3.2028,
+      "theoretical_loss": 4.435918215103694,
+      "tokens_seen": 173015040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736145261963306,
+      "loss": 3.2845,
+      "theoretical_loss": 4.431639172221903,
+      "tokens_seen": 174325760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733780972195952,
+      "loss": 3.1797,
+      "theoretical_loss": 4.427401114202768,
+      "tokens_seen": 175636480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047314166824285987,
+      "loss": 3.1893,
+      "theoretical_loss": 4.4232033469991245,
+      "tokens_seen": 176947200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047290523926612446,
+      "loss": 3.1756,
+      "theoretical_loss": 4.419045193347259,
+      "tokens_seen": 178257920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047266881028938906,
+      "loss": 3.2366,
+      "theoretical_loss": 4.414925992241184,
+      "tokens_seen": 179568640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724323813126537,
+      "loss": 3.1557,
+      "theoretical_loss": 4.410845098427085,
+      "tokens_seen": 180879360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047219595233591826,
+      "loss": 3.1718,
+      "theoretical_loss": 4.406801881917029,
+      "tokens_seen": 182190080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719595233591829,
+      "loss": 3.1319,
+      "theoretical_loss": 4.402795727521044,
+      "tokens_seen": 183500800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047172309438244756,
+      "loss": 3.1649,
+      "theoretical_loss": 4.398826034396784,
+      "tokens_seen": 184811520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714866654057121,
+      "loss": 3.1622,
+      "theoretical_loss": 4.394892215615966,
+      "tokens_seen": 186122240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047125023642897675,
+      "loss": 3.1566,
+      "theoretical_loss": 4.390993697746873,
+      "tokens_seen": 187432960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047101380745224135,
+      "loss": 3.1888,
+      "theoretical_loss": 4.387129920452183,
+      "tokens_seen": 188743680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047077737847550595,
+      "loss": 3.215,
+      "theoretical_loss": 4.383300336101497,
+      "tokens_seen": 190054400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705409494987706,
+      "loss": 3.1893,
+      "theoretical_loss": 4.379504409397913,
+      "tokens_seen": 191365120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703045205220352,
+      "loss": 3.2361,
+      "theoretical_loss": 4.375741617018028,
+      "tokens_seen": 192675840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700680915452998,
+      "loss": 3.2134,
+      "theoretical_loss": 4.372011447264842,
+      "tokens_seen": 193986560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698316625685644,
+      "loss": 3.1681,
+      "theoretical_loss": 4.368313399732967,
+      "tokens_seen": 195297280
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 119577,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.755495548248291,
+      "objective/train/theoretical_loss": 4.364646984985669,
+      "objective/train/tokens_used": 217068000,
+      "theoretical_loss": 4.364646984985669,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046959523359182905,
+      "loss": 3.2212,
+      "theoretical_loss": 4.364646984985669,
+      "tokens_seen": 196608000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693588046150936,
+      "loss": 3.1891,
+      "theoretical_loss": 4.3610117242432125,
+      "tokens_seen": 197918720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046912237563835824,
+      "loss": 3.178,
+      "theoretical_loss": 4.357407149082071,
+      "tokens_seen": 199229440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688859466616229,
+      "loss": 3.1648,
+      "theoretical_loss": 4.353832801144535,
+      "tokens_seen": 200540160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046864951768488744,
+      "loss": 3.173,
+      "theoretical_loss": 4.350288231858301,
+      "tokens_seen": 201850880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684130887081521,
+      "loss": 3.2234,
+      "theoretical_loss": 4.346773002165636,
+      "tokens_seen": 203161600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681766597314167,
+      "loss": 3.1263,
+      "theoretical_loss": 4.343286682261728,
+      "tokens_seen": 204472320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679402307546813,
+      "loss": 3.134,
+      "theoretical_loss": 4.339828851341859,
+      "tokens_seen": 205783040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046770380177794593,
+      "loss": 3.1541,
+      "theoretical_loss": 4.336399097357047,
+      "tokens_seen": 207093760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046746737280121053,
+      "loss": 3.1915,
+      "theoretical_loss": 4.332997016777814,
+      "tokens_seen": 208404480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046723094382447513,
+      "loss": 3.1536,
+      "theoretical_loss": 4.329622214365776,
+      "tokens_seen": 209715200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669945148477397,
+      "loss": 3.1375,
+      "theoretical_loss": 4.326274302952741,
+      "tokens_seen": 211025920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667580858710044,
+      "loss": 3.1703,
+      "theoretical_loss": 4.322952903227008,
+      "tokens_seen": 212336640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466521656894269,
+      "loss": 3.1176,
+      "theoretical_loss": 4.319657643526625,
+      "tokens_seen": 213647360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046628522791753357,
+      "loss": 3.1154,
+      "theoretical_loss": 4.316388159639304,
+      "tokens_seen": 214958080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660487989407982,
+      "loss": 3.1322,
+      "theoretical_loss": 4.313144094608757,
+      "tokens_seen": 216268800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046581236996406277,
+      "loss": 3.1225,
+      "theoretical_loss": 4.3099250985472,
+      "tokens_seen": 217579520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655759409873274,
+      "loss": 3.112,
+      "theoretical_loss": 4.306730828453813,
+      "tokens_seen": 218890240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046533951201059207,
+      "loss": 3.1225,
+      "theoretical_loss": 4.303560948038896,
+      "tokens_seen": 220200960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651030830338566,
+      "loss": 3.1593,
+      "theoretical_loss": 4.300415127553551,
+      "tokens_seen": 221511680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046486665405712127,
+      "loss": 3.1288,
+      "theoretical_loss": 4.297293043624652,
+      "tokens_seen": 222822400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046463022508038586,
+      "loss": 3.1382,
+      "theoretical_loss": 4.294194379094925,
+      "tokens_seen": 224133120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046439379610365046,
+      "loss": 3.1598,
+      "theoretical_loss": 4.291118822867954,
+      "tokens_seen": 225443840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641573671269151,
+      "loss": 3.2182,
+      "theoretical_loss": 4.288066069757916,
+      "tokens_seen": 226754560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639209381501797,
+      "loss": 3.0914,
+      "theoretical_loss": 4.285035820343889,
+      "tokens_seen": 228065280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636845091734443,
+      "loss": 3.1134,
+      "theoretical_loss": 4.28202778082858,
+      "tokens_seen": 229376000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634480801967089,
+      "loss": 3.139,
+      "theoretical_loss": 4.27904166290127,
+      "tokens_seen": 230686720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046321165121997356,
+      "loss": 3.2063,
+      "theoretical_loss": 4.276077183604883,
+      "tokens_seen": 231997440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629752222432381,
+      "loss": 3.2426,
+      "theoretical_loss": 4.2731340652069925,
+      "tokens_seen": 233308160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046273879326650275,
+      "loss": 3.2121,
+      "theoretical_loss": 4.270212035074646,
+      "tokens_seen": 234618880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625023642897674,
+      "loss": 3.1968,
+      "theoretical_loss": 4.267310825552862,
+      "tokens_seen": 235929600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046226593531303195,
+      "loss": 3.1832,
+      "theoretical_loss": 4.264430173846689,
+      "tokens_seen": 237240320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620295063362966,
+      "loss": 3.1976,
+      "theoretical_loss": 4.261569821906684,
+      "tokens_seen": 238551040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617930773595612,
+      "loss": 3.1039,
+      "theoretical_loss": 4.258729516317703,
+      "tokens_seen": 239861760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615566483828258,
+      "loss": 3.1643,
+      "theoretical_loss": 4.255909008190896,
+      "tokens_seen": 241172480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046132021940609044,
+      "loss": 3.1203,
+      "theoretical_loss": 4.253108053058778,
+      "tokens_seen": 242483200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046108379042935504,
+      "loss": 3.089,
+      "theoretical_loss": 4.250326410773302,
+      "tokens_seen": 243793920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046084736145261964,
+      "loss": 3.0638,
+      "theoretical_loss": 4.247563845406793,
+      "tokens_seen": 245104640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046061093247588424,
+      "loss": 3.0637,
+      "theoretical_loss": 4.2448201251556945,
+      "tokens_seen": 246415360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603745034991489,
+      "loss": 3.0402,
+      "theoretical_loss": 4.242095022246987,
+      "tokens_seen": 247726080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601380745224135,
+      "loss": 3.0963,
+      "theoretical_loss": 4.239388312847225,
+      "tokens_seen": 249036800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599016455456781,
+      "loss": 3.0717,
+      "theoretical_loss": 4.236699776974095,
+      "tokens_seen": 250347520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045966521656894274,
+      "loss": 3.1156,
+      "theoretical_loss": 4.234029198410394,
+      "tokens_seen": 251658240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594287875922073,
+      "loss": 3.0526,
+      "theoretical_loss": 4.23137636462039,
+      "tokens_seen": 252968960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045919235861547193,
+      "loss": 3.0776,
+      "theoretical_loss": 4.228741066668439,
+      "tokens_seen": 254279680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589559296387366,
+      "loss": 3.0966,
+      "theoretical_loss": 4.226123099139831,
+      "tokens_seen": 255590400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587195006620011,
+      "loss": 3.1011,
+      "theoretical_loss": 4.223522260063753,
+      "tokens_seen": 256901120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584830716852658,
+      "loss": 3.0844,
+      "theoretical_loss": 4.220938350838335,
+      "tokens_seen": 258211840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582466427085304,
+      "loss": 3.1128,
+      "theoretical_loss": 4.218371176157692,
+      "tokens_seen": 259522560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045801021373179497,
+      "loss": 3.0639,
+      "theoretical_loss": 4.215820543940906,
+      "tokens_seen": 260833280
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 155958,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7153196334838867,
+      "objective/train/theoretical_loss": 4.213286265262889,
+      "objective/train/tokens_used": 282604000,
+      "theoretical_loss": 4.213286265262889,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577737847550596,
+      "loss": 3.1155,
+      "theoretical_loss": 4.213286265262889,
+      "tokens_seen": 262144000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575373557783242,
+      "loss": 3.0747,
+      "theoretical_loss": 4.210768154287058,
+      "tokens_seen": 263454720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573009268015888,
+      "loss": 3.0549,
+      "theoretical_loss": 4.208266028199782,
+      "tokens_seen": 264765440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570644978248534,
+      "loss": 3.0457,
+      "theoretical_loss": 4.205779707146535,
+      "tokens_seen": 266076160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045682806884811807,
+      "loss": 2.9724,
+      "theoretical_loss": 4.203309014169696,
+      "tokens_seen": 267386880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565916398713826,
+      "loss": 3.0444,
+      "theoretical_loss": 4.200853775147967,
+      "tokens_seen": 268697600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045635521089464726,
+      "loss": 3.0419,
+      "theoretical_loss": 4.198413818737336,
+      "tokens_seen": 270008320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561187819179119,
+      "loss": 2.9646,
+      "theoretical_loss": 4.195988976313554,
+      "tokens_seen": 271319040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045588235294117646,
+      "loss": 2.9766,
+      "theoretical_loss": 4.193579081916074,
+      "tokens_seen": 272629760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556459239644411,
+      "loss": 3.0205,
+      "theoretical_loss": 4.191183972193405,
+      "tokens_seen": 273940480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554094949877057,
+      "loss": 2.9421,
+      "theoretical_loss": 4.1888034863498484,
+      "tokens_seen": 275251200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551730660109703,
+      "loss": 2.9571,
+      "theoretical_loss": 4.186437466093564,
+      "tokens_seen": 276561920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045493663703423496,
+      "loss": 3.0165,
+      "theoretical_loss": 4.1840857555859365,
+      "tokens_seen": 277872640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547002080574995,
+      "loss": 2.9436,
+      "theoretical_loss": 4.181748201392195,
+      "tokens_seen": 279183360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045446377908076415,
+      "loss": 2.9791,
+      "theoretical_loss": 4.179424652433255,
+      "tokens_seen": 280494080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045422735010402875,
+      "loss": 3.0287,
+      "theoretical_loss": 4.177114959938745,
+      "tokens_seen": 281804800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045399092112729335,
+      "loss": 3.0399,
+      "theoretical_loss": 4.174818977401182,
+      "tokens_seen": 283115520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000453754492150558,
+      "loss": 3.0741,
+      "theoretical_loss": 4.172536560531271,
+      "tokens_seen": 284426240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004535180631738226,
+      "loss": 2.9846,
+      "theoretical_loss": 4.170267567214273,
+      "tokens_seen": 285736960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004532816341970872,
+      "loss": 3.0432,
+      "theoretical_loss": 4.168011857467442,
+      "tokens_seen": 287047680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004530452052203518,
+      "loss": 3.0537,
+      "theoretical_loss": 4.1657692933984825,
+      "tokens_seen": 288358400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045280877624361644,
+      "loss": 3.0266,
+      "theoretical_loss": 4.163539739164992,
+      "tokens_seen": 289669120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045257234726688104,
+      "loss": 3.0235,
+      "theoretical_loss": 4.161323060934883,
+      "tokens_seen": 290979840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045233591829014564,
+      "loss": 3.0412,
+      "theoretical_loss": 4.159119126847729,
+      "tokens_seen": 292290560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004520994893134103,
+      "loss": 3.0878,
+      "theoretical_loss": 4.1569278069770395,
+      "tokens_seen": 293601280
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045186306033667483,
+      "loss": 3.0076,
+      "theoretical_loss": 4.154748973293404,
+      "tokens_seen": 294912000
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004516266313599395,
+      "loss": 3.0318,
+      "theoretical_loss": 4.152582499628515,
+      "tokens_seen": 296222720
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004513902023832041,
+      "loss": 3.0521,
+      "theoretical_loss": 4.150428261640019,
+      "tokens_seen": 297533440
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004511537734064687,
+      "loss": 3.0571,
+      "theoretical_loss": 4.148286136777182,
+      "tokens_seen": 298844160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045091734442973333,
+      "loss": 3.0267,
+      "theoretical_loss": 4.1461560042473575,
+      "tokens_seen": 300154880
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00045068091545299793,
+      "loss": 3.0734,
+      "theoretical_loss": 4.144037744983214,
+      "tokens_seen": 301465600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004504444864762625,
+      "loss": 2.9788,
+      "theoretical_loss": 4.141931241610721,
+      "tokens_seen": 302776320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004502080574995271,
+      "loss": 3.0159,
+      "theoretical_loss": 4.139836378417861,
+      "tokens_seen": 304087040
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004499716285227918,
+      "loss": 3.0441,
+      "theoretical_loss": 4.1377530413240535,
+      "tokens_seen": 305397760
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044973519954605637,
+      "loss": 3.0352,
+      "theoretical_loss": 4.135681117850265,
+      "tokens_seen": 306708480
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044949877056932097,
+      "loss": 3.0608,
+      "theoretical_loss": 4.1336204970898,
+      "tokens_seen": 308019200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004492623415925856,
+      "loss": 2.9696,
+      "theoretical_loss": 4.131571069679739,
+      "tokens_seen": 309329920
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044902591261585016,
+      "loss": 2.983,
+      "theoretical_loss": 4.129532727773022,
+      "tokens_seen": 310640640
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004487894836391148,
+      "loss": 2.9285,
+      "theoretical_loss": 4.127505365011135,
+      "tokens_seen": 311951360
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044855305466237947,
+      "loss": 2.9094,
+      "theoretical_loss": 4.125488876497428,
+      "tokens_seen": 313262080
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.000448316625685644,
+      "loss": 3.0228,
+      "theoretical_loss": 4.123483158770997,
+      "tokens_seen": 314572800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044808019670890866,
+      "loss": 2.9893,
+      "theoretical_loss": 4.121488109781147,
+      "tokens_seen": 315883520
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044784376773217326,
+      "loss": 2.9421,
+      "theoretical_loss": 4.11950362886242,
+      "tokens_seen": 317194240
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.00044760733875543786,
+      "loss": 2.997,
+      "theoretical_loss": 4.1175296167101525,
+      "tokens_seen": 318504960
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004473709097787025,
+      "loss": 2.9688,
+      "theoretical_loss": 4.115565975356576,
+      "tokens_seen": 319815680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0004471344808019671,
+      "loss": 2.9504,
+      "theoretical_loss": 4.113612608147429,
+      "tokens_seen": 321126400
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004468980518252317,
+      "loss": 2.9478,
+      "theoretical_loss": 4.111669419719063,
+      "tokens_seen": 322437120
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004466616228484963,
+      "loss": 2.9692,
+      "theoretical_loss": 4.109736315976054,
+      "tokens_seen": 323747840
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044642519387176095,
+      "loss": 2.9255,
+      "theoretical_loss": 4.107813204069279,
+      "tokens_seen": 325058560
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004461887648950255,
+      "loss": 2.9303,
+      "theoretical_loss": 4.105899992374464,
+      "tokens_seen": 326369280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.6313309751661036,
+      "debugging/distinct-1-grams": 0.7812735170334136,
+      "debugging/distinct-2-grams": 0.962251014076992,
+      "debugging/entropy-1-grams": 6.495179970030579,
+      "debugging/entropy-2-grams": 7.614784181109197,
+      "debugging/length": 541.0,
+      "debugging/num_segments": 29,
+      "epoch": 0.12,
+      "objective/train/docs_used": 191983,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.092848062515259,
+      "objective/train/theoretical_loss": 4.103996590471185,
+      "objective/train/tokens_used": 348140000,
+      "theoretical_loss": 4.103996590471185,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044595233591829015,
+      "loss": 2.9576,
+      "theoretical_loss": 4.103996590471185,
+      "tokens_seen": 327680000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004457159069415548,
+      "loss": 2.9776,
+      "theoretical_loss": 4.102102909122308,
+      "tokens_seen": 328990720
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044547947796481934,
+      "loss": 2.9632,
+      "theoretical_loss": 4.1002188602538645,
+      "tokens_seen": 330301440
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000445243048988084,
+      "loss": 2.9661,
+      "theoretical_loss": 4.098344356935348,
+      "tokens_seen": 331612160
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004450066200113486,
+      "loss": 3.0443,
+      "theoretical_loss": 4.096479313360415,
+      "tokens_seen": 332922880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004447701910346132,
+      "loss": 2.9526,
+      "theoretical_loss": 4.094623644827995,
+      "tokens_seen": 334233600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044453376205787784,
+      "loss": 2.9707,
+      "theoretical_loss": 4.092777267723779,
+      "tokens_seen": 335544320
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044429733308114244,
+      "loss": 2.9704,
+      "theoretical_loss": 4.0909400995021,
+      "tokens_seen": 336855040
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044406090410440704,
+      "loss": 2.9488,
+      "theoretical_loss": 4.08911205866818,
+      "tokens_seen": 338165760
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044382447512767163,
+      "loss": 2.9595,
+      "theoretical_loss": 4.087293064760739,
+      "tokens_seen": 339476480
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004435880461509363,
+      "loss": 2.9613,
+      "theoretical_loss": 4.0854830383349565,
+      "tokens_seen": 340787200
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004433516171742009,
+      "loss": 2.934,
+      "theoretical_loss": 4.083681900945785,
+      "tokens_seen": 342097920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004431151881974655,
+      "loss": 2.9695,
+      "theoretical_loss": 4.081889575131589,
+      "tokens_seen": 343408640
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.00044287875922073013,
+      "loss": 2.984,
+      "theoretical_loss": 4.080105984398123,
+      "tokens_seen": 344719360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004426423302439947,
+      "loss": 2.9943,
+      "theoretical_loss": 4.078331053202827,
+      "tokens_seen": 346030080
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004424059012672593,
+      "loss": 2.9157,
+      "theoretical_loss": 4.076564706939434,
+      "tokens_seen": 347340800
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.000442169472290524,
+      "loss": 2.9823,
+      "theoretical_loss": 4.074806871922881,
+      "tokens_seen": 348651520
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0004419330433137885,
+      "loss": 2.8975,
+      "theoretical_loss": 4.0730574753745294,
+      "tokens_seen": 349962240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004416966143370532,
+      "loss": 2.9424,
+      "theoretical_loss": 4.071316445407663,
+      "tokens_seen": 351272960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044146018536031777,
+      "loss": 2.915,
+      "theoretical_loss": 4.06958371101328,
+      "tokens_seen": 352583680
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044122375638358237,
+      "loss": 2.9268,
+      "theoretical_loss": 4.067859202046162,
+      "tokens_seen": 353894400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.000440987327406847,
+      "loss": 3.0175,
+      "theoretical_loss": 4.066142849211214,
+      "tokens_seen": 355205120
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004407508984301116,
+      "loss": 3.01,
+      "theoretical_loss": 4.064434584050067,
+      "tokens_seen": 356515840
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004405144694533762,
+      "loss": 3.0178,
+      "theoretical_loss": 4.062734338927946,
+      "tokens_seen": 357826560
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004402780404766408,
+      "loss": 2.9272,
+      "theoretical_loss": 4.061042047020785,
+      "tokens_seen": 359137280
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00044004161149990546,
+      "loss": 2.9365,
+      "theoretical_loss": 4.0593576423025945,
+      "tokens_seen": 360448000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043980518252317,
+      "loss": 2.979,
+      "theoretical_loss": 4.057681059533071,
+      "tokens_seen": 361758720
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043956875354643466,
+      "loss": 2.9836,
+      "theoretical_loss": 4.056012234245436,
+      "tokens_seen": 363069440
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004393323245696993,
+      "loss": 2.9923,
+      "theoretical_loss": 4.05435110273452,
+      "tokens_seen": 364380160
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043909589559296385,
+      "loss": 2.9818,
+      "theoretical_loss": 4.052697602045054,
+      "tokens_seen": 365690880
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004388594666162285,
+      "loss": 2.999,
+      "theoretical_loss": 4.0510516699602,
+      "tokens_seen": 367001600
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004386230376394931,
+      "loss": 2.9427,
+      "theoretical_loss": 4.049413244990284,
+      "tokens_seen": 368312320
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004383866086627577,
+      "loss": 2.9272,
+      "theoretical_loss": 4.04778226636175,
+      "tokens_seen": 369623040
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043815017968602235,
+      "loss": 2.9664,
+      "theoretical_loss": 4.046158674006312,
+      "tokens_seen": 370933760
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043791375070928695,
+      "loss": 3.0413,
+      "theoretical_loss": 4.044542408550312,
+      "tokens_seen": 372244480
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043767732173255155,
+      "loss": 2.9541,
+      "theoretical_loss": 4.042933411304277,
+      "tokens_seen": 373555200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.00043744089275581614,
+      "loss": 2.9418,
+      "theoretical_loss": 4.041331624252669,
+      "tokens_seen": 374865920
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004372044637790808,
+      "loss": 2.9744,
+      "theoretical_loss": 4.039736990043815,
+      "tokens_seen": 376176640
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 0.0004369680348023454,
+      "loss": 2.9218,
+      "theoretical_loss": 4.038149451980039,
+      "tokens_seen": 377487360
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043673160582561,
+      "loss": 2.9285,
+      "theoretical_loss": 4.0365689540079535,
+      "tokens_seen": 378798080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043649517684887464,
+      "loss": 2.9238,
+      "theoretical_loss": 4.034995440708947,
+      "tokens_seen": 380108800
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004362587478721392,
+      "loss": 2.9433,
+      "theoretical_loss": 4.033428857289831,
+      "tokens_seen": 381419520
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043602231889540384,
+      "loss": 2.9385,
+      "theoretical_loss": 4.031869149573668,
+      "tokens_seen": 382730240
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004357858899186685,
+      "loss": 2.9034,
+      "theoretical_loss": 4.030316263990749,
+      "tokens_seen": 384040960
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043554946094193303,
+      "loss": 2.9103,
+      "theoretical_loss": 4.028770147569752,
+      "tokens_seen": 385351680
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004353130319651977,
+      "loss": 2.9029,
+      "theoretical_loss": 4.0272307479290435,
+      "tokens_seen": 386662400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004350766029884623,
+      "loss": 2.8958,
+      "theoretical_loss": 4.025698013268144,
+      "tokens_seen": 387973120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004348401740117269,
+      "loss": 2.9768,
+      "theoretical_loss": 4.024171892359345,
+      "tokens_seen": 389283840
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043460374503499153,
+      "loss": 2.9533,
+      "theoretical_loss": 4.0226523345394645,
+      "tokens_seen": 390594560
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043436731605825613,
+      "loss": 2.9034,
+      "theoretical_loss": 4.021139289701768,
+      "tokens_seen": 391905280
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 227836,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.685603618621826,
+      "objective/train/theoretical_loss": 4.019632708288012,
+      "objective/train/tokens_used": 413676000,
+      "theoretical_loss": 4.019632708288012,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004341308870815207,
+      "loss": 2.893,
+      "theoretical_loss": 4.019632708288012,
+      "tokens_seen": 393216000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004338944581047853,
+      "loss": 2.9011,
+      "theoretical_loss": 4.018132541280638,
+      "tokens_seen": 394526720
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043365802912805,
+      "loss": 2.9062,
+      "theoretical_loss": 4.016638740195099,
+      "tokens_seen": 395837440
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004334216001513145,
+      "loss": 2.9063,
+      "theoretical_loss": 4.015151257072327,
+      "tokens_seen": 397148160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043318517117457917,
+      "loss": 2.9453,
+      "theoretical_loss": 4.013670044471318,
+      "tokens_seen": 398458880
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004329487421978438,
+      "loss": 2.9131,
+      "theoretical_loss": 4.012195055461859,
+      "tokens_seen": 399769600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.00043271231322110837,
+      "loss": 2.9323,
+      "theoretical_loss": 4.010726243617374,
+      "tokens_seen": 401080320
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.000432475884244373,
+      "loss": 2.9073,
+      "theoretical_loss": 4.009263563007895,
+      "tokens_seen": 402391040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004322394552676376,
+      "loss": 2.9628,
+      "theoretical_loss": 4.007806968193149,
+      "tokens_seen": 403701760
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0004320030262909022,
+      "loss": 2.9356,
+      "theoretical_loss": 4.006356414215775,
+      "tokens_seen": 405012480
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043176659731416686,
+      "loss": 2.8839,
+      "theoretical_loss": 4.004911856594639,
+      "tokens_seen": 406323200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004315301683374314,
+      "loss": 2.9369,
+      "theoretical_loss": 4.00347325131828,
+      "tokens_seen": 407633920
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043129373936069606,
+      "loss": 2.9701,
+      "theoretical_loss": 4.002040554838458,
+      "tokens_seen": 408944640
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043105731038396066,
+      "loss": 2.9915,
+      "theoretical_loss": 4.000613724063812,
+      "tokens_seen": 410255360
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00043082088140722525,
+      "loss": 2.9283,
+      "theoretical_loss": 3.9991927163536243,
+      "tokens_seen": 411566080
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004305844524304899,
+      "loss": 2.9208,
+      "theoretical_loss": 3.997777489511691,
+      "tokens_seen": 412876800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004303480234537545,
+      "loss": 3.0067,
+      "theoretical_loss": 3.9963680017802945,
+      "tokens_seen": 414187520
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004301115944770191,
+      "loss": 3.0753,
+      "theoretical_loss": 3.9949642118342794,
+      "tokens_seen": 415498240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004298751655002837,
+      "loss": 2.9815,
+      "theoretical_loss": 3.993566078775218,
+      "tokens_seen": 416808960
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042963873652354835,
+      "loss": 3.0226,
+      "theoretical_loss": 3.9921735621256835,
+      "tokens_seen": 418119680
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042940230754681295,
+      "loss": 2.998,
+      "theoretical_loss": 3.990786621823611,
+      "tokens_seen": 419430400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042916587857007754,
+      "loss": 3.0086,
+      "theoretical_loss": 3.9894052182167536,
+      "tokens_seen": 420741120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004289294495933422,
+      "loss": 2.9585,
+      "theoretical_loss": 3.988029312057227,
+      "tokens_seen": 422051840
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042869302061660674,
+      "loss": 2.9746,
+      "theoretical_loss": 3.986658864496149,
+      "tokens_seen": 423362560
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004284565916398714,
+      "loss": 2.9944,
+      "theoretical_loss": 3.985293837078359,
+      "tokens_seen": 424673280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.000428220162663136,
+      "loss": 2.9603,
+      "theoretical_loss": 3.9839341917372293,
+      "tokens_seen": 425984000
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004279837336864006,
+      "loss": 2.9428,
+      "theoretical_loss": 3.9825798907895598,
+      "tokens_seen": 427294720
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042774730470966524,
+      "loss": 3.0057,
+      "theoretical_loss": 3.9812308969305508,
+      "tokens_seen": 428605440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042751087573292984,
+      "loss": 2.9675,
+      "theoretical_loss": 3.9798871732288617,
+      "tokens_seen": 429916160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042727444675619443,
+      "loss": 2.9784,
+      "theoretical_loss": 3.9785486831217467,
+      "tokens_seen": 431226880
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00042703801777945903,
+      "loss": 2.9846,
+      "theoretical_loss": 3.9772153904102705,
+      "tokens_seen": 432537600
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.0004268015888027237,
+      "loss": 2.9641,
+      "theoretical_loss": 3.9758872592545957,
+      "tokens_seen": 433848320
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004265651598259883,
+      "loss": 2.9532,
+      "theoretical_loss": 3.974564254169353,
+      "tokens_seen": 435159040
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004263287308492529,
+      "loss": 2.916,
+      "theoretical_loss": 3.9732463400190756,
+      "tokens_seen": 436469760
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042609230187251753,
+      "loss": 2.9596,
+      "theoretical_loss": 3.9719334820137124,
+      "tokens_seen": 437780480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042585587289578207,
+      "loss": 3.0018,
+      "theoretical_loss": 3.9706256457042133,
+      "tokens_seen": 439091200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004256194439190467,
+      "loss": 2.9557,
+      "theoretical_loss": 3.969322796978175,
+      "tokens_seen": 440401920
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004253830149423114,
+      "loss": 2.872,
+      "theoretical_loss": 3.968024902055565,
+      "tokens_seen": 441712640
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004251465859655759,
+      "loss": 2.9665,
+      "theoretical_loss": 3.966731927484508,
+      "tokens_seen": 443023360
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042491015698884057,
+      "loss": 2.9158,
+      "theoretical_loss": 3.9654438401371372,
+      "tokens_seen": 444334080
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042467372801210517,
+      "loss": 2.9577,
+      "theoretical_loss": 3.9641606072055113,
+      "tokens_seen": 445644800
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042443729903536976,
+      "loss": 2.8971,
+      "theoretical_loss": 3.9628821961975955,
+      "tokens_seen": 446955520
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004242008700586344,
+      "loss": 2.9398,
+      "theoretical_loss": 3.9616085749333028,
+      "tokens_seen": 448266240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.000423964441081899,
+      "loss": 2.9665,
+      "theoretical_loss": 3.9603397115406005,
+      "tokens_seen": 449576960
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004237280121051636,
+      "loss": 2.8911,
+      "theoretical_loss": 3.9590755744516684,
+      "tokens_seen": 450887680
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004234915831284282,
+      "loss": 2.921,
+      "theoretical_loss": 3.9578161323991243,
+      "tokens_seen": 452198400
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042325515415169286,
+      "loss": 2.904,
+      "theoretical_loss": 3.9565613544123046,
+      "tokens_seen": 453509120
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004230187251749574,
+      "loss": 2.894,
+      "theoretical_loss": 3.955311209813596,
+      "tokens_seen": 454819840
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042278229619822206,
+      "loss": 2.8959,
+      "theoretical_loss": 3.9540656682148327,
+      "tokens_seen": 456130560
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004225458672214867,
+      "loss": 2.8807,
+      "theoretical_loss": 3.952824699513741,
+      "tokens_seen": 457441280
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 262792,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.0986132621765137,
+      "objective/train/theoretical_loss": 3.951588273890438,
+      "objective/train/tokens_used": 479212000,
+      "theoretical_loss": 3.951588273890438,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.00042230943824475125,
+      "loss": 2.9211,
+      "theoretical_loss": 3.951588273890438,
+      "tokens_seen": 458752000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004220730092680159,
+      "loss": 2.9374,
+      "theoretical_loss": 3.950356361803988,
+      "tokens_seen": 460062720
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 0.0004218365802912805,
+      "loss": 2.9011,
+      "theoretical_loss": 3.949128933989004,
+      "tokens_seen": 461373440
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004216001513145451,
+      "loss": 2.894,
+      "theoretical_loss": 3.9479059614523058,
+      "tokens_seen": 462684160
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042136372233780975,
+      "loss": 2.8602,
+      "theoretical_loss": 3.9466874154696234,
+      "tokens_seen": 463994880
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042112729336107435,
+      "loss": 2.8006,
+      "theoretical_loss": 3.945473267582355,
+      "tokens_seen": 465305600
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042089086438433894,
+      "loss": 2.8434,
+      "theoretical_loss": 3.9442634895943662,
+      "tokens_seen": 466616320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00042065443540760354,
+      "loss": 2.8697,
+      "theoretical_loss": 3.943058053568841,
+      "tokens_seen": 467927040
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004204180064308682,
+      "loss": 2.9221,
+      "theoretical_loss": 3.94185693182518,
+      "tokens_seen": 469237760
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004201815774541328,
+      "loss": 2.9025,
+      "theoretical_loss": 3.9406600969359413,
+      "tokens_seen": 470548480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004199451484773974,
+      "loss": 2.8998,
+      "theoretical_loss": 3.9394675217238277,
+      "tokens_seen": 471859200
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041970871950066204,
+      "loss": 2.8875,
+      "theoretical_loss": 3.9382791792587195,
+      "tokens_seen": 473169920
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004194722905239266,
+      "loss": 2.9253,
+      "theoretical_loss": 3.9370950428547475,
+      "tokens_seen": 474480640
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041923586154719123,
+      "loss": 2.8817,
+      "theoretical_loss": 3.9359150860674106,
+      "tokens_seen": 475791360
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004189994325704559,
+      "loss": 2.7852,
+      "theoretical_loss": 3.9347392826907353,
+      "tokens_seen": 477102080
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041876300359372043,
+      "loss": 2.8527,
+      "theoretical_loss": 3.933567606754476,
+      "tokens_seen": 478412800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004185265746169851,
+      "loss": 2.8854,
+      "theoretical_loss": 3.9324000325213557,
+      "tokens_seen": 479723520
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004182901456402497,
+      "loss": 2.8954,
+      "theoretical_loss": 3.931236534484346,
+      "tokens_seen": 481034240
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004180537166635143,
+      "loss": 2.8211,
+      "theoretical_loss": 3.930077087363987,
+      "tokens_seen": 482344960
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041781728768677893,
+      "loss": 2.8287,
+      "theoretical_loss": 3.928921666105744,
+      "tokens_seen": 483655680
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004175808587100435,
+      "loss": 2.8199,
+      "theoretical_loss": 3.9277702458774058,
+      "tokens_seen": 484966400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004173444297333081,
+      "loss": 2.8797,
+      "theoretical_loss": 3.926622802066514,
+      "tokens_seen": 486277120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.0004171080007565727,
+      "loss": 2.8932,
+      "theoretical_loss": 3.925479310277835,
+      "tokens_seen": 487587840
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00041687157177983737,
+      "loss": 2.8838,
+      "theoretical_loss": 3.924339746330864,
+      "tokens_seen": 488898560
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004166351428031019,
+      "loss": 2.869,
+      "theoretical_loss": 3.923204086257365,
+      "tokens_seen": 490209280
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041639871382636657,
+      "loss": 2.9711,
+      "theoretical_loss": 3.9220723062989444,
+      "tokens_seen": 491520000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004161622848496312,
+      "loss": 2.9418,
+      "theoretical_loss": 3.920944382904665,
+      "tokens_seen": 492830720
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041592585587289576,
+      "loss": 2.8859,
+      "theoretical_loss": 3.919820292728682,
+      "tokens_seen": 494141440
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004156894268961604,
+      "loss": 2.9165,
+      "theoretical_loss": 3.918700012627923,
+      "tokens_seen": 495452160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.000415452997919425,
+      "loss": 2.9237,
+      "theoretical_loss": 3.9175835196597957,
+      "tokens_seen": 496762880
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004152165689426896,
+      "loss": 2.9148,
+      "theoretical_loss": 3.9164707910799264,
+      "tokens_seen": 498073600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041498013996595426,
+      "loss": 2.7991,
+      "theoretical_loss": 3.9153618043399296,
+      "tokens_seen": 499384320
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041474371098921886,
+      "loss": 2.8596,
+      "theoretical_loss": 3.9142565370852136,
+      "tokens_seen": 500695040
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041450728201248346,
+      "loss": 2.8489,
+      "theoretical_loss": 3.9131549671528116,
+      "tokens_seen": 502005760
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041427085303574805,
+      "loss": 2.8473,
+      "theoretical_loss": 3.912057072569243,
+      "tokens_seen": 503316480
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004140344240590127,
+      "loss": 2.8647,
+      "theoretical_loss": 3.910962831548404,
+      "tokens_seen": 504627200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004137979950822773,
+      "loss": 2.8317,
+      "theoretical_loss": 3.909872222489491,
+      "tokens_seen": 505937920
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004135615661055419,
+      "loss": 2.8661,
+      "theoretical_loss": 3.908785223974948,
+      "tokens_seen": 507248640
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041332513712880655,
+      "loss": 2.8311,
+      "theoretical_loss": 3.9077018147684415,
+      "tokens_seen": 508559360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004130887081520711,
+      "loss": 2.8473,
+      "theoretical_loss": 3.906621973812867,
+      "tokens_seen": 509870080
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041285227917533575,
+      "loss": 2.8414,
+      "theoretical_loss": 3.9055456802283812,
+      "tokens_seen": 511180800
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004126158501986004,
+      "loss": 2.8146,
+      "theoretical_loss": 3.904472913310456,
+      "tokens_seen": 512491520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.00041237942122186494,
+      "loss": 2.8244,
+      "theoretical_loss": 3.9034036525279645,
+      "tokens_seen": 513802240
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004121429922451296,
+      "loss": 2.8241,
+      "theoretical_loss": 3.902337877521294,
+      "tokens_seen": 515112960
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004119065632683942,
+      "loss": 2.806,
+      "theoretical_loss": 3.9012755681004756,
+      "tokens_seen": 516423680
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0004116701342916588,
+      "loss": 2.7768,
+      "theoretical_loss": 3.9002167042433475,
+      "tokens_seen": 517734400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041143370531492344,
+      "loss": 2.8269,
+      "theoretical_loss": 3.899161266093741,
+      "tokens_seen": 519045120
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041119727633818804,
+      "loss": 2.8854,
+      "theoretical_loss": 3.8981092339596852,
+      "tokens_seen": 520355840
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041096084736145263,
+      "loss": 2.7923,
+      "theoretical_loss": 3.8970605883116436,
+      "tokens_seen": 521666560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00041072441838471723,
+      "loss": 2.8312,
+      "theoretical_loss": 3.896015309780767,
+      "tokens_seen": 522977280
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 298912,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.896383285522461,
+      "objective/train/theoretical_loss": 3.8949733791571752,
+      "objective/train/tokens_used": 544748000,
+      "theoretical_loss": 3.8949733791571752,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004104879894079819,
+      "loss": 2.7988,
+      "theoretical_loss": 3.8949733791571752,
+      "tokens_seen": 524288000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004102515604312464,
+      "loss": 2.8545,
+      "theoretical_loss": 3.8939347773882584,
+      "tokens_seen": 525598720
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004100151314545111,
+      "loss": 2.8124,
+      "theoretical_loss": 3.892899485576998,
+      "tokens_seen": 526909440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040977870247777573,
+      "loss": 2.8338,
+      "theoretical_loss": 3.8918674849803176,
+      "tokens_seen": 528220160
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004095422735010403,
+      "loss": 2.801,
+      "theoretical_loss": 3.890838757007447,
+      "tokens_seen": 529530880
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004093058445243049,
+      "loss": 2.8481,
+      "theoretical_loss": 3.889813283218313,
+      "tokens_seen": 530841600
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004090694155475695,
+      "loss": 2.7885,
+      "theoretical_loss": 3.888791045321949,
+      "tokens_seen": 532152320
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004088329865708341,
+      "loss": 2.8324,
+      "theoretical_loss": 3.887772025174925,
+      "tokens_seen": 533463040
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040859655759409877,
+      "loss": 2.8701,
+      "theoretical_loss": 3.8867562047797986,
+      "tokens_seen": 534773760
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040836012861736337,
+      "loss": 2.852,
+      "theoretical_loss": 3.885743566283587,
+      "tokens_seen": 536084480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004081355210894647,
+      "loss": 2.9002,
+      "theoretical_loss": 3.8847340919762576,
+      "tokens_seen": 537395200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040789909211272937,
+      "loss": 2.866,
+      "theoretical_loss": 3.883727764289236,
+      "tokens_seen": 538705920
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040766266313599397,
+      "loss": 2.895,
+      "theoretical_loss": 3.8827245657939375,
+      "tokens_seen": 540016640
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040742623415925856,
+      "loss": 2.9149,
+      "theoretical_loss": 3.8817244792003143,
+      "tokens_seen": 541327360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00040718980518252316,
+      "loss": 2.8674,
+      "theoretical_loss": 3.8807274873554216,
+      "tokens_seen": 542638080
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004069533762057878,
+      "loss": 2.8413,
+      "theoretical_loss": 3.8797335732420035,
+      "tokens_seen": 543948800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.0004067169472290524,
+      "loss": 2.913,
+      "theoretical_loss": 3.878742719977094,
+      "tokens_seen": 545259520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.000406480518252317,
+      "loss": 2.7836,
+      "theoretical_loss": 3.877754910810639,
+      "tokens_seen": 546570240
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004062440892755816,
+      "loss": 2.8622,
+      "theoretical_loss": 3.876770129124133,
+      "tokens_seen": 547880960
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004060076602988462,
+      "loss": 2.8448,
+      "theoretical_loss": 3.8757883584292774,
+      "tokens_seen": 549191680
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040577123132211085,
+      "loss": 2.8386,
+      "theoretical_loss": 3.8748095823666473,
+      "tokens_seen": 550502400
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040553480234537545,
+      "loss": 2.9041,
+      "theoretical_loss": 3.8738337847043836,
+      "tokens_seen": 551813120
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040529837336864005,
+      "loss": 2.8887,
+      "theoretical_loss": 3.8728609493368986,
+      "tokens_seen": 553123840
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004050619443919047,
+      "loss": 2.8391,
+      "theoretical_loss": 3.8718910602835943,
+      "tokens_seen": 554434560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040482551541516924,
+      "loss": 2.8807,
+      "theoretical_loss": 3.870924101687603,
+      "tokens_seen": 555745280
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004045890864384339,
+      "loss": 2.8711,
+      "theoretical_loss": 3.869960057814537,
+      "tokens_seen": 557056000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040435265746169855,
+      "loss": 2.8464,
+      "theoretical_loss": 3.868998913051261,
+      "tokens_seen": 558366720
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004041162284849631,
+      "loss": 2.878,
+      "theoretical_loss": 3.868040651904671,
+      "tokens_seen": 559677440
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040387979950822774,
+      "loss": 2.8201,
+      "theoretical_loss": 3.867085259000496,
+      "tokens_seen": 560988160
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040364337053149234,
+      "loss": 2.8307,
+      "theoretical_loss": 3.8661327190821124,
+      "tokens_seen": 562298880
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040340694155475694,
+      "loss": 2.8384,
+      "theoretical_loss": 3.8651830170093673,
+      "tokens_seen": 563609600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004031705125780216,
+      "loss": 2.8647,
+      "theoretical_loss": 3.864236137757427,
+      "tokens_seen": 564920320
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004029340836012862,
+      "loss": 2.8788,
+      "theoretical_loss": 3.863292066415628,
+      "tokens_seen": 566231040
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004026976546245508,
+      "loss": 2.8845,
+      "theoretical_loss": 3.862350788186353,
+      "tokens_seen": 567541760
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004024612256478154,
+      "loss": 2.9096,
+      "theoretical_loss": 3.8614122883839093,
+      "tokens_seen": 568852480
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040222479667108003,
+      "loss": 2.9111,
+      "theoretical_loss": 3.8604765524334312,
+      "tokens_seen": 570163200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.0004019883676943446,
+      "loss": 2.9061,
+      "theoretical_loss": 3.8595435658697888,
+      "tokens_seen": 571473920
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 0.00040175193871760923,
+      "loss": 2.8971,
+      "theoretical_loss": 3.858613314336514,
+      "tokens_seen": 572784640
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004015155097408739,
+      "loss": 2.9225,
+      "theoretical_loss": 3.8576857835847367,
+      "tokens_seen": 574095360
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004012790807641384,
+      "loss": 2.8995,
+      "theoretical_loss": 3.8567609594721333,
+      "tokens_seen": 575406080
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004010426517874031,
+      "loss": 2.8884,
+      "theoretical_loss": 3.855838827961894,
+      "tokens_seen": 576716800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004008062228106677,
+      "loss": 2.9238,
+      "theoretical_loss": 3.8549193751216935,
+      "tokens_seen": 578027520
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00040056979383393227,
+      "loss": 2.9618,
+      "theoretical_loss": 3.854002587122681,
+      "tokens_seen": 579338240
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004003333648571969,
+      "loss": 2.9991,
+      "theoretical_loss": 3.853088450238478,
+      "tokens_seen": 580648960
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0004000969358804615,
+      "loss": 2.8924,
+      "theoretical_loss": 3.8521769508441905,
+      "tokens_seen": 581959680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003998605069037261,
+      "loss": 2.8746,
+      "theoretical_loss": 3.8512680754154323,
+      "tokens_seen": 583270400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003996240779269907,
+      "loss": 2.9294,
+      "theoretical_loss": 3.850361810527361,
+      "tokens_seen": 584581120
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039938764895025537,
+      "loss": 2.8919,
+      "theoretical_loss": 3.8494581428537202,
+      "tokens_seen": 585891840
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039915121997351996,
+      "loss": 2.8493,
+      "theoretical_loss": 3.8485570591659006,
+      "tokens_seen": 587202560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039891479099678456,
+      "loss": 2.874,
+      "theoretical_loss": 3.847658546332007,
+      "tokens_seen": 588513280
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 333293,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 3.176788806915283,
+      "objective/train/theoretical_loss": 3.8467625913159385,
+      "objective/train/tokens_used": 610284000,
+      "theoretical_loss": 3.8467625913159385,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003986783620200492,
+      "loss": 2.8767,
+      "theoretical_loss": 3.8467625913159385,
+      "tokens_seen": 589824000
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039844193304331376,
+      "loss": 2.9135,
+      "theoretical_loss": 3.845869181176477,
+      "tokens_seen": 591134720
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003982055040665784,
+      "loss": 2.845,
+      "theoretical_loss": 3.844978303066389,
+      "tokens_seen": 592445440
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039796907508984306,
+      "loss": 2.873,
+      "theoretical_loss": 3.8440899442315395,
+      "tokens_seen": 593756160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003977326461131076,
+      "loss": 2.8381,
+      "theoretical_loss": 3.843204092010007,
+      "tokens_seen": 595066880
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039749621713637225,
+      "loss": 2.8171,
+      "theoretical_loss": 3.8423207338312233,
+      "tokens_seen": 596377600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039725978815963685,
+      "loss": 2.8206,
+      "theoretical_loss": 3.841439857215109,
+      "tokens_seen": 597688320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.00039702335918290145,
+      "loss": 2.849,
+      "theoretical_loss": 3.84056144977123,
+      "tokens_seen": 598999040
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003967869302061661,
+      "loss": 2.8171,
+      "theoretical_loss": 3.839685499197956,
+      "tokens_seen": 600309760
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0003965505012294307,
+      "loss": 2.8053,
+      "theoretical_loss": 3.8388119932816327,
+      "tokens_seen": 601620480
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003963140722526953,
+      "loss": 2.7998,
+      "theoretical_loss": 3.8379409198957637,
+      "tokens_seen": 602931200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003960776432759599,
+      "loss": 2.7579,
+      "theoretical_loss": 3.837072267000199,
+      "tokens_seen": 604241920
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039584121429922455,
+      "loss": 2.8011,
+      "theoretical_loss": 3.8362060226403347,
+      "tokens_seen": 605552640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003956047853224891,
+      "loss": 2.8101,
+      "theoretical_loss": 3.835342174946324,
+      "tokens_seen": 606863360
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039536835634575374,
+      "loss": 2.8487,
+      "theoretical_loss": 3.834480712132291,
+      "tokens_seen": 608174080
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003951319273690184,
+      "loss": 2.8284,
+      "theoretical_loss": 3.833621622495561,
+      "tokens_seen": 609484800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039489549839228294,
+      "loss": 2.8678,
+      "theoretical_loss": 3.8327648944158907,
+      "tokens_seen": 610795520
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003946590694155476,
+      "loss": 2.8471,
+      "theoretical_loss": 3.8319105163547187,
+      "tokens_seen": 612106240
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003944226404388122,
+      "loss": 2.8486,
+      "theoretical_loss": 3.8310584768544125,
+      "tokens_seen": 613416960
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003941862114620768,
+      "loss": 2.8515,
+      "theoretical_loss": 3.8302087645375327,
+      "tokens_seen": 614727680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039394978248534143,
+      "loss": 2.836,
+      "theoretical_loss": 3.829361368106099,
+      "tokens_seen": 616038400
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039371335350860603,
+      "loss": 2.7877,
+      "theoretical_loss": 3.8285162763408707,
+      "tokens_seen": 617349120
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039347692453187063,
+      "loss": 2.8079,
+      "theoretical_loss": 3.827673478100631,
+      "tokens_seen": 618659840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003932404955551352,
+      "loss": 2.7835,
+      "theoretical_loss": 3.8268329623214803,
+      "tokens_seen": 619970560
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003930040665783999,
+      "loss": 2.8813,
+      "theoretical_loss": 3.8259947180161373,
+      "tokens_seen": 621281280
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003927676376016645,
+      "loss": 2.8321,
+      "theoretical_loss": 3.8251587342732485,
+      "tokens_seen": 622592000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039253120862492907,
+      "loss": 2.8772,
+      "theoretical_loss": 3.824325000256705,
+      "tokens_seen": 623902720
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003922947796481937,
+      "loss": 2.7669,
+      "theoretical_loss": 3.8234935052049663,
+      "tokens_seen": 625213440
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039205835067145827,
+      "loss": 2.7804,
+      "theoretical_loss": 3.8226642384303933,
+      "tokens_seen": 626524160
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.0003918219216947229,
+      "loss": 2.7675,
+      "theoretical_loss": 3.821837189318586,
+      "tokens_seen": 627834880
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039158549271798757,
+      "loss": 2.8066,
+      "theoretical_loss": 3.82101234732773,
+      "tokens_seen": 629145600
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003913490637412521,
+      "loss": 2.813,
+      "theoretical_loss": 3.820189701987953,
+      "tokens_seen": 630456320
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039111263476451677,
+      "loss": 2.8604,
+      "theoretical_loss": 3.819369242900682,
+      "tokens_seen": 631767040
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039087620578778136,
+      "loss": 2.7827,
+      "theoretical_loss": 3.8185509597380105,
+      "tokens_seen": 633077760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039063977681104596,
+      "loss": 2.8927,
+      "theoretical_loss": 3.8177348422420785,
+      "tokens_seen": 634388480
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00039040334783431056,
+      "loss": 2.7982,
+      "theoretical_loss": 3.8169208802244476,
+      "tokens_seen": 635699200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003901669188575752,
+      "loss": 2.8225,
+      "theoretical_loss": 3.8161090635654937,
+      "tokens_seen": 637009920
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003899304898808398,
+      "loss": 2.8336,
+      "theoretical_loss": 3.815299382213799,
+      "tokens_seen": 638320640
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003896940609041044,
+      "loss": 2.8672,
+      "theoretical_loss": 3.8144918261855536,
+      "tokens_seen": 639631360
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038945763192736906,
+      "loss": 2.886,
+      "theoretical_loss": 3.813686385563966,
+      "tokens_seen": 640942080
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003892212029506336,
+      "loss": 2.8648,
+      "theoretical_loss": 3.812883050498675,
+      "tokens_seen": 642252800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038898477397389825,
+      "loss": 2.8726,
+      "theoretical_loss": 3.812081811205168,
+      "tokens_seen": 643563520
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003887483449971629,
+      "loss": 2.8236,
+      "theoretical_loss": 3.811282657964214,
+      "tokens_seen": 644874240
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038851191602042745,
+      "loss": 2.8662,
+      "theoretical_loss": 3.8104855811212905,
+      "tokens_seen": 646184960
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003882754870436921,
+      "loss": 2.8414,
+      "theoretical_loss": 3.8096905710860254,
+      "tokens_seen": 647495680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003880390580669567,
+      "loss": 2.8482,
+      "theoretical_loss": 3.808897618331641,
+      "tokens_seen": 648806400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003878026290902213,
+      "loss": 2.8853,
+      "theoretical_loss": 3.8081067133944053,
+      "tokens_seen": 650117120
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038756620011348594,
+      "loss": 2.8402,
+      "theoretical_loss": 3.8073178468730884,
+      "tokens_seen": 651427840
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038732977113675054,
+      "loss": 2.8161,
+      "theoretical_loss": 3.806531009428425,
+      "tokens_seen": 652738560
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038709334216001514,
+      "loss": 2.9016,
+      "theoretical_loss": 3.805746191782582,
+      "tokens_seen": 654049280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5624010415402148,
+      "debugging/distinct-1-grams": 0.7656679022066195,
+      "debugging/distinct-2-grams": 0.9502794404107235,
+      "debugging/entropy-1-grams": 6.252376785281594,
+      "debugging/entropy-2-grams": 7.364077311746257,
+      "debugging/length": 469.7037037037037,
+      "debugging/num_segments": 27,
+      "epoch": 0.23,
+      "objective/train/docs_used": 369378,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.6431469917297363,
+      "objective/train/theoretical_loss": 3.8049633847186337,
+      "objective/train/tokens_used": 675820000,
+      "theoretical_loss": 3.8049633847186337,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00038685691318327974,
+      "loss": 2.8082,
+      "theoretical_loss": 3.8049633847186337,
+      "tokens_seen": 655360000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.0003866204842065444,
+      "loss": 2.7756,
+      "theoretical_loss": 3.80418257908004,
+      "tokens_seen": 656670720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.000386384055229809,
+      "loss": 2.824,
+      "theoretical_loss": 3.803403765770131,
+      "tokens_seen": 657981440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003861476262530736,
+      "loss": 2.792,
+      "theoretical_loss": 3.802626935751598,
+      "tokens_seen": 659292160
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038591119727633824,
+      "loss": 2.8023,
+      "theoretical_loss": 3.801852080045988,
+      "tokens_seen": 660602880
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003856747682996028,
+      "loss": 2.8259,
+      "theoretical_loss": 3.801079189733206,
+      "tokens_seen": 661913600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038543833932286743,
+      "loss": 2.7732,
+      "theoretical_loss": 3.8003082559510197,
+      "tokens_seen": 663224320
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003852019103461321,
+      "loss": 2.8362,
+      "theoretical_loss": 3.799539269894571,
+      "tokens_seen": 664535040
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003849654813693966,
+      "loss": 2.8237,
+      "theoretical_loss": 3.798772222815893,
+      "tokens_seen": 665845760
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003847290523926613,
+      "loss": 2.8672,
+      "theoretical_loss": 3.79800710602343,
+      "tokens_seen": 667156480
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003844926234159259,
+      "loss": 2.8139,
+      "theoretical_loss": 3.7972439108815657,
+      "tokens_seen": 668467200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038425619443919047,
+      "loss": 2.7977,
+      "theoretical_loss": 3.7964826288101508,
+      "tokens_seen": 669777920
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038401976546245507,
+      "loss": 2.792,
+      "theoretical_loss": 3.795723251284044,
+      "tokens_seen": 671088640
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003837833364857197,
+      "loss": 2.8414,
+      "theoretical_loss": 3.7949657698326487,
+      "tokens_seen": 672399360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003835469075089843,
+      "loss": 2.8634,
+      "theoretical_loss": 3.79421017603946,
+      "tokens_seen": 673710080
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003833104785322489,
+      "loss": 2.8814,
+      "theoretical_loss": 3.7934564615416146,
+      "tokens_seen": 675020800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003830740495555135,
+      "loss": 2.8822,
+      "theoretical_loss": 3.7927046180294455,
+      "tokens_seen": 676331520
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003828376205787781,
+      "loss": 2.8902,
+      "theoretical_loss": 3.7919546372460418,
+      "tokens_seen": 677642240
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038260119160204276,
+      "loss": 2.8806,
+      "theoretical_loss": 3.7912065109868127,
+      "tokens_seen": 678952960
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038236476262530736,
+      "loss": 2.9011,
+      "theoretical_loss": 3.790460231099053,
+      "tokens_seen": 680263680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038212833364857196,
+      "loss": 2.8691,
+      "theoretical_loss": 3.789715789481521,
+      "tokens_seen": 681574400
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003818919046718366,
+      "loss": 2.8599,
+      "theoretical_loss": 3.788973178084011,
+      "tokens_seen": 682885120
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00038165547569510115,
+      "loss": 2.8565,
+      "theoretical_loss": 3.788232388906935,
+      "tokens_seen": 684195840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.0003814190467183658,
+      "loss": 2.9027,
+      "theoretical_loss": 3.7874934140009096,
+      "tokens_seen": 685506560
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038118261774163046,
+      "loss": 2.9221,
+      "theoretical_loss": 3.786756245466346,
+      "tokens_seen": 686817280
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000380946188764895,
+      "loss": 2.8943,
+      "theoretical_loss": 3.7860208754530422,
+      "tokens_seen": 688128000
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038070975978815965,
+      "loss": 2.8787,
+      "theoretical_loss": 3.785287296159782,
+      "tokens_seen": 689438720
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038047333081142425,
+      "loss": 2.8168,
+      "theoretical_loss": 3.784555499833936,
+      "tokens_seen": 690749440
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038023690183468885,
+      "loss": 2.82,
+      "theoretical_loss": 3.7838254787710683,
+      "tokens_seen": 692060160
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003800004728579535,
+      "loss": 2.8135,
+      "theoretical_loss": 3.7830972253145463,
+      "tokens_seen": 693370880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003797640438812181,
+      "loss": 2.8702,
+      "theoretical_loss": 3.7823707318551536,
+      "tokens_seen": 694681600
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003795276149044827,
+      "loss": 2.8203,
+      "theoretical_loss": 3.7816459908307074,
+      "tokens_seen": 695992320
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003792911859277473,
+      "loss": 2.8083,
+      "theoretical_loss": 3.7809229947256826,
+      "tokens_seen": 697303040
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037905475695101194,
+      "loss": 2.8808,
+      "theoretical_loss": 3.7802017360708327,
+      "tokens_seen": 698613760
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003788183279742765,
+      "loss": 2.8116,
+      "theoretical_loss": 3.779482207442821,
+      "tokens_seen": 699924480
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037858189899754114,
+      "loss": 2.8466,
+      "theoretical_loss": 3.778764401463854,
+      "tokens_seen": 701235200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003783454700208058,
+      "loss": 2.7396,
+      "theoretical_loss": 3.778048310801316,
+      "tokens_seen": 702545920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037810904104407033,
+      "loss": 2.7644,
+      "theoretical_loss": 3.7773339281674074,
+      "tokens_seen": 703856640
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000377872612067335,
+      "loss": 2.8114,
+      "theoretical_loss": 3.7766212463187916,
+      "tokens_seen": 705167360
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003776361830905996,
+      "loss": 2.8038,
+      "theoretical_loss": 3.7759102580562383,
+      "tokens_seen": 706478080
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003773997541138642,
+      "loss": 2.8995,
+      "theoretical_loss": 3.7752009562242757,
+      "tokens_seen": 707788800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037716332513712883,
+      "loss": 2.7762,
+      "theoretical_loss": 3.7744933337108444,
+      "tokens_seen": 709099520
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00037692689616039343,
+      "loss": 2.7827,
+      "theoretical_loss": 3.773787383446951,
+      "tokens_seen": 710410240
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000376690467183658,
+      "loss": 2.8006,
+      "theoretical_loss": 3.773083098406333,
+      "tokens_seen": 711720960
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0003764540382069226,
+      "loss": 2.851,
+      "theoretical_loss": 3.7723804716051195,
+      "tokens_seen": 713031680
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003762176092301873,
+      "loss": 2.8083,
+      "theoretical_loss": 3.7716794961014983,
+      "tokens_seen": 714342400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037598118025345187,
+      "loss": 2.8337,
+      "theoretical_loss": 3.770980164995387,
+      "tokens_seen": 715653120
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037574475127671647,
+      "loss": 2.8571,
+      "theoretical_loss": 3.7702824714281062,
+      "tokens_seen": 716963840
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003755083222999811,
+      "loss": 2.7931,
+      "theoretical_loss": 3.7695864085820547,
+      "tokens_seen": 718274560
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037527189332324566,
+      "loss": 2.7179,
+      "theoretical_loss": 3.7688919696803898,
+      "tokens_seen": 719585280
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 405200,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.6444921493530273,
+      "objective/train/theoretical_loss": 3.76819914798671,
+      "objective/train/tokens_used": 741356000,
+      "theoretical_loss": 3.76819914798671,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003750354643465103,
+      "loss": 2.7525,
+      "theoretical_loss": 3.76819914798671,
+      "tokens_seen": 720896000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037479903536977497,
+      "loss": 2.831,
+      "theoretical_loss": 3.7675079368047415,
+      "tokens_seen": 722206720
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003745626063930395,
+      "loss": 2.767,
+      "theoretical_loss": 3.7668183294780246,
+      "tokens_seen": 723517440
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037432617741630416,
+      "loss": 2.7648,
+      "theoretical_loss": 3.7661303193896076,
+      "tokens_seen": 724828160
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037408974843956876,
+      "loss": 2.7352,
+      "theoretical_loss": 3.76544389996174,
+      "tokens_seen": 726138880
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037385331946283336,
+      "loss": 2.7489,
+      "theoretical_loss": 3.7647590646555713,
+      "tokens_seen": 727449600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000373616890486098,
+      "loss": 2.7566,
+      "theoretical_loss": 3.7640758069708493,
+      "tokens_seen": 728760320
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003733804615093626,
+      "loss": 2.7534,
+      "theoretical_loss": 3.763394120445626,
+      "tokens_seen": 730071040
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003731440325326272,
+      "loss": 2.7722,
+      "theoretical_loss": 3.7627139986559595,
+      "tokens_seen": 731381760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003729076035558918,
+      "loss": 2.7775,
+      "theoretical_loss": 3.7620354352156293,
+      "tokens_seen": 732692480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037267117457915645,
+      "loss": 2.7509,
+      "theoretical_loss": 3.7613584237758415,
+      "tokens_seen": 734003200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.000372434745602421,
+      "loss": 2.8231,
+      "theoretical_loss": 3.7606829580249457,
+      "tokens_seen": 735313920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037219831662568565,
+      "loss": 2.8032,
+      "theoretical_loss": 3.760009031688152,
+      "tokens_seen": 736624640
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003719618876489503,
+      "loss": 2.7746,
+      "theoretical_loss": 3.7593366385272513,
+      "tokens_seen": 737935360
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00037172545867221484,
+      "loss": 2.8175,
+      "theoretical_loss": 3.758665772340336,
+      "tokens_seen": 739246080
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003714890296954795,
+      "loss": 2.8326,
+      "theoretical_loss": 3.7579964269615247,
+      "tokens_seen": 740556800
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0003712526007187441,
+      "loss": 2.8052,
+      "theoretical_loss": 3.7573285962606917,
+      "tokens_seen": 741867520
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003710161717420087,
+      "loss": 2.8092,
+      "theoretical_loss": 3.7566622741431956,
+      "tokens_seen": 743178240
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037077974276527334,
+      "loss": 2.8054,
+      "theoretical_loss": 3.75599745454961,
+      "tokens_seen": 744488960
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037054331378853794,
+      "loss": 2.8374,
+      "theoretical_loss": 3.7553341314554625,
+      "tokens_seen": 745799680
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037030688481180254,
+      "loss": 2.7937,
+      "theoretical_loss": 3.754672298870968,
+      "tokens_seen": 747110400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00037007045583506713,
+      "loss": 2.8163,
+      "theoretical_loss": 3.754011950840771,
+      "tokens_seen": 748421120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003698340268583318,
+      "loss": 2.7632,
+      "theoretical_loss": 3.753353081443688,
+      "tokens_seen": 749731840
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003695975978815964,
+      "loss": 2.8264,
+      "theoretical_loss": 3.752695684792449,
+      "tokens_seen": 751042560
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000369361168904861,
+      "loss": 2.7693,
+      "theoretical_loss": 3.752039755033449,
+      "tokens_seen": 752353280
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036912473992812563,
+      "loss": 2.7898,
+      "theoretical_loss": 3.751385286346493,
+      "tokens_seen": 753664000
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003688883109513902,
+      "loss": 2.7758,
+      "theoretical_loss": 3.7507322729445525,
+      "tokens_seen": 754974720
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003686518819746548,
+      "loss": 2.6996,
+      "theoretical_loss": 3.7500807090735115,
+      "tokens_seen": 756285440
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003684154529979195,
+      "loss": 2.7918,
+      "theoretical_loss": 3.749430589011933,
+      "tokens_seen": 757596160
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.000368179024021184,
+      "loss": 2.7646,
+      "theoretical_loss": 3.748781907070807,
+      "tokens_seen": 758906880
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003679425950444487,
+      "loss": 2.7615,
+      "theoretical_loss": 3.7481346575933197,
+      "tokens_seen": 760217600
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036770616606771327,
+      "loss": 2.8018,
+      "theoretical_loss": 3.747488834954609,
+      "tokens_seen": 761528320
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036746973709097787,
+      "loss": 2.7596,
+      "theoretical_loss": 3.7468444335615363,
+      "tokens_seen": 762839040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036723330811424247,
+      "loss": 2.8004,
+      "theoretical_loss": 3.746201447852446,
+      "tokens_seen": 764149760
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003669968791375071,
+      "loss": 2.7333,
+      "theoretical_loss": 3.745559872296941,
+      "tokens_seen": 765460480
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003667604501607717,
+      "loss": 2.6807,
+      "theoretical_loss": 3.7449197013956503,
+      "tokens_seen": 766771200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0003665240211840363,
+      "loss": 2.7509,
+      "theoretical_loss": 3.744280929680005,
+      "tokens_seen": 768081920
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00036628759220730096,
+      "loss": 2.7345,
+      "theoretical_loss": 3.743643551712009,
+      "tokens_seen": 769392640
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003660511632305655,
+      "loss": 2.7554,
+      "theoretical_loss": 3.7430075620840215,
+      "tokens_seen": 770703360
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036581473425383016,
+      "loss": 2.7471,
+      "theoretical_loss": 3.7423729554185337,
+      "tokens_seen": 772014080
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003655783052770948,
+      "loss": 2.7502,
+      "theoretical_loss": 3.7417397263679493,
+      "tokens_seen": 773324800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036534187630035935,
+      "loss": 2.776,
+      "theoretical_loss": 3.7411078696143694,
+      "tokens_seen": 774635520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000365105447323624,
+      "loss": 2.8418,
+      "theoretical_loss": 3.740477379869378,
+      "tokens_seen": 775946240
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003648690183468886,
+      "loss": 2.7558,
+      "theoretical_loss": 3.739848251873827,
+      "tokens_seen": 777256960
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003646325893701532,
+      "loss": 2.7823,
+      "theoretical_loss": 3.739220480397626,
+      "tokens_seen": 778567680
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036439616039341785,
+      "loss": 2.7753,
+      "theoretical_loss": 3.7385940602395347,
+      "tokens_seen": 779878400
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036415973141668245,
+      "loss": 2.7528,
+      "theoretical_loss": 3.737968986226954,
+      "tokens_seen": 781189120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036392330243994705,
+      "loss": 2.8357,
+      "theoretical_loss": 3.7373452532157208,
+      "tokens_seen": 782499840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036368687346321164,
+      "loss": 2.836,
+      "theoretical_loss": 3.736722856089904,
+      "tokens_seen": 783810560
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003634504444864763,
+      "loss": 2.8279,
+      "theoretical_loss": 3.736101789761605,
+      "tokens_seen": 785121280
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 440553,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.734668493270874,
+      "objective/train/theoretical_loss": 3.735482049170753,
+      "objective/train/tokens_used": 806892000,
+      "theoretical_loss": 3.735482049170753,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003632140155097409,
+      "loss": 2.8129,
+      "theoretical_loss": 3.735482049170753,
+      "tokens_seen": 786432000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003629775865330055,
+      "loss": 2.8144,
+      "theoretical_loss": 3.734863629284911,
+      "tokens_seen": 787742720
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036274115755627014,
+      "loss": 2.8018,
+      "theoretical_loss": 3.7342465250990777,
+      "tokens_seen": 789053440
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003625047285795347,
+      "loss": 2.7858,
+      "theoretical_loss": 3.7336307316354933,
+      "tokens_seen": 790364160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036226829960279934,
+      "loss": 2.8347,
+      "theoretical_loss": 3.7330162439434442,
+      "tokens_seen": 791674880
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.000362031870626064,
+      "loss": 2.8298,
+      "theoretical_loss": 3.7324030570990736,
+      "tokens_seen": 792985600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00036179544164932853,
+      "loss": 2.8485,
+      "theoretical_loss": 3.7317911662051912,
+      "tokens_seen": 794296320
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003615590126725932,
+      "loss": 2.7562,
+      "theoretical_loss": 3.731180566391086,
+      "tokens_seen": 795607040
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0003613225836958578,
+      "loss": 2.8055,
+      "theoretical_loss": 3.7305712528123367,
+      "tokens_seen": 796917760
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003610861547191224,
+      "loss": 2.8396,
+      "theoretical_loss": 3.729963220650631,
+      "tokens_seen": 798228480
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000360849725742387,
+      "loss": 2.8725,
+      "theoretical_loss": 3.729356465113578,
+      "tokens_seen": 799539200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00036061329676565163,
+      "loss": 2.8118,
+      "theoretical_loss": 3.7287509814345317,
+      "tokens_seen": 800849920
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003603768677889162,
+      "loss": 2.8708,
+      "theoretical_loss": 3.7281467648724043,
+      "tokens_seen": 802160640
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003601404388121808,
+      "loss": 2.7887,
+      "theoretical_loss": 3.727543810711495,
+      "tokens_seen": 803471360
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003599040098354455,
+      "loss": 2.7755,
+      "theoretical_loss": 3.7269421142613055,
+      "tokens_seen": 804782080
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035966758085871,
+      "loss": 2.8072,
+      "theoretical_loss": 3.7263416708563715,
+      "tokens_seen": 806092800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035943115188197467,
+      "loss": 2.8006,
+      "theoretical_loss": 3.725742475856083,
+      "tokens_seen": 807403520
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035919472290523927,
+      "loss": 2.86,
+      "theoretical_loss": 3.7251445246445147,
+      "tokens_seen": 808714240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035895829392850387,
+      "loss": 2.8171,
+      "theoretical_loss": 3.724547812630255,
+      "tokens_seen": 810024960
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003587218649517685,
+      "loss": 2.8207,
+      "theoretical_loss": 3.7239523352462354,
+      "tokens_seen": 811335680
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035848543597503306,
+      "loss": 2.8229,
+      "theoretical_loss": 3.7233580879495616,
+      "tokens_seen": 812646400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003582490069982977,
+      "loss": 2.7929,
+      "theoretical_loss": 3.72276506622135,
+      "tokens_seen": 813957120
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035801257802156236,
+      "loss": 2.8218,
+      "theoretical_loss": 3.722173265566559,
+      "tokens_seen": 815267840
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003577761490448269,
+      "loss": 2.8068,
+      "theoretical_loss": 3.7215826815138264,
+      "tokens_seen": 816578560
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003575515415169283,
+      "loss": 2.8181,
+      "theoretical_loss": 3.720993309615307,
+      "tokens_seen": 817889280
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035731511254019296,
+      "loss": 2.8018,
+      "theoretical_loss": 3.720405145446512,
+      "tokens_seen": 819200000
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003570786835634575,
+      "loss": 2.8138,
+      "theoretical_loss": 3.719818184606148,
+      "tokens_seen": 820510720
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035684225458672216,
+      "loss": 2.7894,
+      "theoretical_loss": 3.719232422715959,
+      "tokens_seen": 821821440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035660582560998675,
+      "loss": 2.7918,
+      "theoretical_loss": 3.7186478554205693,
+      "tokens_seen": 823132160
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00035636939663325135,
+      "loss": 2.7905,
+      "theoretical_loss": 3.718064478387329,
+      "tokens_seen": 824442880
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.000356132967656516,
+      "loss": 2.7324,
+      "theoretical_loss": 3.717482287306157,
+      "tokens_seen": 825753600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003558965386797806,
+      "loss": 2.8108,
+      "theoretical_loss": 3.7169012778893897,
+      "tokens_seen": 827064320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003556601097030452,
+      "loss": 2.8041,
+      "theoretical_loss": 3.71632144587163,
+      "tokens_seen": 828375040
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003554236807263098,
+      "loss": 2.7985,
+      "theoretical_loss": 3.7157427870095936,
+      "tokens_seen": 829685760
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035518725174957445,
+      "loss": 2.7984,
+      "theoretical_loss": 3.7151652970819615,
+      "tokens_seen": 830996480
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035495082277283904,
+      "loss": 2.7258,
+      "theoretical_loss": 3.714588971889232,
+      "tokens_seen": 832307200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035471439379610364,
+      "loss": 2.7483,
+      "theoretical_loss": 3.7140138072535738,
+      "tokens_seen": 833617920
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003544779648193683,
+      "loss": 2.7361,
+      "theoretical_loss": 3.7134397990186794,
+      "tokens_seen": 834928640
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035424153584263284,
+      "loss": 2.7957,
+      "theoretical_loss": 3.7128669430496215,
+      "tokens_seen": 836239360
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003540051068658975,
+      "loss": 2.7877,
+      "theoretical_loss": 3.71229523523271,
+      "tokens_seen": 837550080
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035376867788916214,
+      "loss": 2.7284,
+      "theoretical_loss": 3.7117246714753485,
+      "tokens_seen": 838860800
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003535322489124267,
+      "loss": 2.7212,
+      "theoretical_loss": 3.7111552477058947,
+      "tokens_seen": 840171520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035329581993569134,
+      "loss": 2.7321,
+      "theoretical_loss": 3.7105869598735195,
+      "tokens_seen": 841482240
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035305939095895593,
+      "loss": 2.7422,
+      "theoretical_loss": 3.71001980394807,
+      "tokens_seen": 842792960
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035282296198222053,
+      "loss": 2.7512,
+      "theoretical_loss": 3.709453775919931,
+      "tokens_seen": 844103680
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003525865330054852,
+      "loss": 2.7337,
+      "theoretical_loss": 3.7088888717998865,
+      "tokens_seen": 845414400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003523501040287498,
+      "loss": 2.7225,
+      "theoretical_loss": 3.70832508761899,
+      "tokens_seen": 846725120
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003521136750520144,
+      "loss": 2.781,
+      "theoretical_loss": 3.707762419428425,
+      "tokens_seen": 848035840
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.000351877246075279,
+      "loss": 2.763,
+      "theoretical_loss": 3.7072008632993745,
+      "tokens_seen": 849346560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003516408170985436,
+      "loss": 2.7247,
+      "theoretical_loss": 3.7066404153228873,
+      "tokens_seen": 850657280
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 477081,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.682365655899048,
+      "objective/train/theoretical_loss": 3.7060810716097503,
+      "objective/train/tokens_used": 872428000,
+      "theoretical_loss": 3.7060810716097503,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00035140438812180817,
+      "loss": 2.7227,
+      "theoretical_loss": 3.7060810716097503,
+      "tokens_seen": 851968000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0003511679591450728,
+      "loss": 2.7242,
+      "theoretical_loss": 3.7055228282903556,
+      "tokens_seen": 853278720
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035093153016833747,
+      "loss": 2.7308,
+      "theoretical_loss": 3.704965681514573,
+      "tokens_seen": 854589440
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.000350695101191602,
+      "loss": 2.7703,
+      "theoretical_loss": 3.7044096274516214,
+      "tokens_seen": 855900160
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035045867221486667,
+      "loss": 2.7385,
+      "theoretical_loss": 3.7038546622899435,
+      "tokens_seen": 857210880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00035022224323813127,
+      "loss": 2.761,
+      "theoretical_loss": 3.70330078223708,
+      "tokens_seen": 858521600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034998581426139586,
+      "loss": 2.7265,
+      "theoretical_loss": 3.7027479835195427,
+      "tokens_seen": 859832320
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003497493852846605,
+      "loss": 2.7332,
+      "theoretical_loss": 3.702196262382694,
+      "tokens_seen": 861143040
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003495129563079251,
+      "loss": 2.7538,
+      "theoretical_loss": 3.701645615090622,
+      "tokens_seen": 862453760
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003492765273311897,
+      "loss": 2.7039,
+      "theoretical_loss": 3.7010960379260185,
+      "tokens_seen": 863764480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003490400983544543,
+      "loss": 2.6704,
+      "theoretical_loss": 3.7005475271900616,
+      "tokens_seen": 865075200
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034880366937771896,
+      "loss": 2.7704,
+      "theoretical_loss": 3.7000000792022916,
+      "tokens_seen": 866385920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034856724040098356,
+      "loss": 2.7561,
+      "theoretical_loss": 3.699453690300495,
+      "tokens_seen": 867696640
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034833081142424815,
+      "loss": 2.7399,
+      "theoretical_loss": 3.698908356840586,
+      "tokens_seen": 869007360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003480943824475128,
+      "loss": 2.7376,
+      "theoretical_loss": 3.6983640751964897,
+      "tokens_seen": 870318080
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034785795347077735,
+      "loss": 2.7489,
+      "theoretical_loss": 3.697820841760025,
+      "tokens_seen": 871628800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.000347621524494042,
+      "loss": 2.7743,
+      "theoretical_loss": 3.697278652940791,
+      "tokens_seen": 872939520
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034738509551730665,
+      "loss": 2.7589,
+      "theoretical_loss": 3.6967375051660527,
+      "tokens_seen": 874250240
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0003471486665405712,
+      "loss": 2.7634,
+      "theoretical_loss": 3.6961973948806266,
+      "tokens_seen": 875560960
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034691223756383585,
+      "loss": 2.7023,
+      "theoretical_loss": 3.6956583185467693,
+      "tokens_seen": 876871680
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034667580858710044,
+      "loss": 2.701,
+      "theoretical_loss": 3.6951202726440666,
+      "tokens_seen": 878182400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034643937961036504,
+      "loss": 2.7733,
+      "theoretical_loss": 3.6945832536693204,
+      "tokens_seen": 879493120
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00034620295063362964,
+      "loss": 2.8073,
+      "theoretical_loss": 3.6940472581364423,
+      "tokens_seen": 880803840
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003459665216568943,
+      "loss": 2.7698,
+      "theoretical_loss": 3.6935122825763433,
+      "tokens_seen": 882114560
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003457300926801589,
+      "loss": 2.761,
+      "theoretical_loss": 3.692978323536824,
+      "tokens_seen": 883425280
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003454936637034235,
+      "loss": 2.8038,
+      "theoretical_loss": 3.6924453775824704,
+      "tokens_seen": 884736000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034525723472668814,
+      "loss": 2.7369,
+      "theoretical_loss": 3.6919134412945462,
+      "tokens_seen": 886046720
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003450208057499527,
+      "loss": 2.7994,
+      "theoretical_loss": 3.691382511270887,
+      "tokens_seen": 887357440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034478437677321733,
+      "loss": 2.7174,
+      "theoretical_loss": 3.6908525841257953,
+      "tokens_seen": 888668160
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000344547947796482,
+      "loss": 2.729,
+      "theoretical_loss": 3.6903236564899395,
+      "tokens_seen": 889978880
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034431151881974653,
+      "loss": 2.7324,
+      "theoretical_loss": 3.6897957250102467,
+      "tokens_seen": 891289600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003440750898430112,
+      "loss": 2.7016,
+      "theoretical_loss": 3.6892687863498024,
+      "tokens_seen": 892600320
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003438386608662758,
+      "loss": 2.7708,
+      "theoretical_loss": 3.6887428371877506,
+      "tokens_seen": 893911040
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003436022318895404,
+      "loss": 2.7147,
+      "theoretical_loss": 3.688217874219191,
+      "tokens_seen": 895221760
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.000343365802912805,
+      "loss": 2.7911,
+      "theoretical_loss": 3.6876938941550783,
+      "tokens_seen": 896532480
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003431293739360696,
+      "loss": 2.759,
+      "theoretical_loss": 3.6871708937221275,
+      "tokens_seen": 897843200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003428929449593342,
+      "loss": 2.7408,
+      "theoretical_loss": 3.686648869662709,
+      "tokens_seen": 899153920
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003426565159825988,
+      "loss": 2.7732,
+      "theoretical_loss": 3.6861278187347573,
+      "tokens_seen": 900464640
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034242008700586347,
+      "loss": 2.7327,
+      "theoretical_loss": 3.6856077377116696,
+      "tokens_seen": 901775360
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034218365802912807,
+      "loss": 2.6724,
+      "theoretical_loss": 3.685088623382213,
+      "tokens_seen": 903086080
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034194722905239266,
+      "loss": 2.7484,
+      "theoretical_loss": 3.6845704725504254,
+      "tokens_seen": 904396800
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003417108000756573,
+      "loss": 2.7001,
+      "theoretical_loss": 3.684053282035526,
+      "tokens_seen": 905707520
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034147437109892186,
+      "loss": 2.7018,
+      "theoretical_loss": 3.6835370486718153,
+      "tokens_seen": 907018240
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0003412379421221865,
+      "loss": 2.718,
+      "theoretical_loss": 3.6830217693085876,
+      "tokens_seen": 908328960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00034100151314545116,
+      "loss": 2.6959,
+      "theoretical_loss": 3.682507440810034,
+      "tokens_seen": 909639680
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003407650841687157,
+      "loss": 2.7146,
+      "theoretical_loss": 3.6819940600551533,
+      "tokens_seen": 910950400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034052865519198036,
+      "loss": 2.7645,
+      "theoretical_loss": 3.6814816239376595,
+      "tokens_seen": 912261120
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034029222621524496,
+      "loss": 2.8214,
+      "theoretical_loss": 3.6809701293658934,
+      "tokens_seen": 913571840
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00034005579723850955,
+      "loss": 2.8067,
+      "theoretical_loss": 3.6804595732627297,
+      "tokens_seen": 914882560
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033981936826177415,
+      "loss": 2.7421,
+      "theoretical_loss": 3.6799499525654915,
+      "tokens_seen": 916193280
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 513033,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.9703891277313232,
+      "objective/train/theoretical_loss": 3.6794412642258583,
+      "objective/train/tokens_used": 937964000,
+      "theoretical_loss": 3.6794412642258583,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003395829392850388,
+      "loss": 2.7474,
+      "theoretical_loss": 3.6794412642258583,
+      "tokens_seen": 917504000
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003393465103083034,
+      "loss": 2.7983,
+      "theoretical_loss": 3.6789335052097814,
+      "tokens_seen": 918814720
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000339110081331568,
+      "loss": 2.7745,
+      "theoretical_loss": 3.6784266724973946,
+      "tokens_seen": 920125440
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033887365235483265,
+      "loss": 2.7142,
+      "theoretical_loss": 3.6779207630829296,
+      "tokens_seen": 921436160
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003386372233780972,
+      "loss": 2.717,
+      "theoretical_loss": 3.677415773974629,
+      "tokens_seen": 922746880
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033840079440136184,
+      "loss": 2.7773,
+      "theoretical_loss": 3.6769117021946625,
+      "tokens_seen": 924057600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003381643654246265,
+      "loss": 2.75,
+      "theoretical_loss": 3.676408544779039,
+      "tokens_seen": 925368320
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033792793644789104,
+      "loss": 2.7044,
+      "theoretical_loss": 3.6759062987775293,
+      "tokens_seen": 926679040
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003376915074711557,
+      "loss": 2.7295,
+      "theoretical_loss": 3.6754049612535753,
+      "tokens_seen": 927989760
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003374550784944203,
+      "loss": 2.7905,
+      "theoretical_loss": 3.6749045292842126,
+      "tokens_seen": 929300480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0003372186495176849,
+      "loss": 2.7737,
+      "theoretical_loss": 3.674404999959987,
+      "tokens_seen": 930611200
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033698222054094954,
+      "loss": 2.7701,
+      "theoretical_loss": 3.6739063703848718,
+      "tokens_seen": 931921920
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033674579156421413,
+      "loss": 2.7763,
+      "theoretical_loss": 3.6734086376761907,
+      "tokens_seen": 933232640
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033650936258747873,
+      "loss": 2.7574,
+      "theoretical_loss": 3.6729117989645332,
+      "tokens_seen": 934543360
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00033627293361074333,
+      "loss": 2.7159,
+      "theoretical_loss": 3.6724158513936773,
+      "tokens_seen": 935854080
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.000336036504634008,
+      "loss": 2.7697,
+      "theoretical_loss": 3.6719207921205115,
+      "tokens_seen": 937164800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003358000756572726,
+      "loss": 2.7389,
+      "theoretical_loss": 3.6714266183149533,
+      "tokens_seen": 938475520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003355636466805372,
+      "loss": 2.7541,
+      "theoretical_loss": 3.670933327159875,
+      "tokens_seen": 939786240
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033532721770380183,
+      "loss": 2.7769,
+      "theoretical_loss": 3.670440915851024,
+      "tokens_seen": 941096960
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033509078872706637,
+      "loss": 2.7759,
+      "theoretical_loss": 3.6699493815969464,
+      "tokens_seen": 942407680
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.000334854359750331,
+      "loss": 2.751,
+      "theoretical_loss": 3.6694587216189114,
+      "tokens_seen": 943718400
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033461793077359557,
+      "loss": 2.7233,
+      "theoretical_loss": 3.6689689331508366,
+      "tokens_seen": 945029120
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003343815017968602,
+      "loss": 2.7527,
+      "theoretical_loss": 3.6684800134392104,
+      "tokens_seen": 946339840
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033414507282012487,
+      "loss": 2.7004,
+      "theoretical_loss": 3.6679919597430213,
+      "tokens_seen": 947650560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003339086438433894,
+      "loss": 2.728,
+      "theoretical_loss": 3.6675047693336795,
+      "tokens_seen": 948961280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033367221486665406,
+      "loss": 2.7098,
+      "theoretical_loss": 3.6670184394949477,
+      "tokens_seen": 950272000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033343578588991866,
+      "loss": 2.8082,
+      "theoretical_loss": 3.6665329675228646,
+      "tokens_seen": 951582720
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033319935691318326,
+      "loss": 2.7153,
+      "theoretical_loss": 3.666048350725677,
+      "tokens_seen": 952893440
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003329629279364479,
+      "loss": 2.7102,
+      "theoretical_loss": 3.6655645864237627,
+      "tokens_seen": 954204160
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003327264989597125,
+      "loss": 2.6854,
+      "theoretical_loss": 3.6650816719495625,
+      "tokens_seen": 955514880
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003324900699829771,
+      "loss": 2.677,
+      "theoretical_loss": 3.6645996046475107,
+      "tokens_seen": 956825600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003322536410062417,
+      "loss": 2.694,
+      "theoretical_loss": 3.664118381873961,
+      "tokens_seen": 958136320
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033201721202950635,
+      "loss": 2.6956,
+      "theoretical_loss": 3.66363800099712,
+      "tokens_seen": 959447040
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033178078305277095,
+      "loss": 2.6896,
+      "theoretical_loss": 3.6631584593969757,
+      "tokens_seen": 960757760
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033154435407603555,
+      "loss": 2.676,
+      "theoretical_loss": 3.6626797544652305,
+      "tokens_seen": 962068480
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0003313079250993002,
+      "loss": 2.661,
+      "theoretical_loss": 3.6622018836052317,
+      "tokens_seen": 963379200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00033107149612256474,
+      "loss": 2.6784,
+      "theoretical_loss": 3.6617248442319053,
+      "tokens_seen": 964689920
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033084688859466615,
+      "loss": 2.7124,
+      "theoretical_loss": 3.661248633771687,
+      "tokens_seen": 966000640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033062228106676755,
+      "loss": 2.7365,
+      "theoretical_loss": 3.6607732496624568,
+      "tokens_seen": 967311360
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033038585209003215,
+      "loss": 2.7163,
+      "theoretical_loss": 3.660298689353471,
+      "tokens_seen": 968622080
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00033014942311329675,
+      "loss": 2.6828,
+      "theoretical_loss": 3.659824950305297,
+      "tokens_seen": 969932800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003299129941365614,
+      "loss": 2.749,
+      "theoretical_loss": 3.6593520299897513,
+      "tokens_seen": 971243520
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.000329676565159826,
+      "loss": 2.7235,
+      "theoretical_loss": 3.6588799258898277,
+      "tokens_seen": 972554240
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003294401361830906,
+      "loss": 2.7695,
+      "theoretical_loss": 3.65840863549964,
+      "tokens_seen": 973864960
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032920370720635524,
+      "loss": 2.7298,
+      "theoretical_loss": 3.6579381563243527,
+      "tokens_seen": 975175680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003289672782296198,
+      "loss": 2.7102,
+      "theoretical_loss": 3.6574684858801207,
+      "tokens_seen": 976486400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032873084925288444,
+      "loss": 2.6791,
+      "theoretical_loss": 3.656999621694025,
+      "tokens_seen": 977797120
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003284944202761491,
+      "loss": 2.6754,
+      "theoretical_loss": 3.65653156130401,
+      "tokens_seen": 979107840
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032825799129941363,
+      "loss": 2.6584,
+      "theoretical_loss": 3.6560643022588217,
+      "tokens_seen": 980418560
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003280215623226783,
+      "loss": 2.7284,
+      "theoretical_loss": 3.655597842117945,
+      "tokens_seen": 981729280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.5483677763402848,
+      "debugging/distinct-1-grams": 0.7892743979655746,
+      "debugging/distinct-2-grams": 0.9538096028345523,
+      "debugging/entropy-1-grams": 6.171578287824214,
+      "debugging/entropy-2-grams": 7.07806555195415,
+      "debugging/length": 523.5294117647059,
+      "debugging/num_segments": 17,
+      "epoch": 0.35,
+      "objective/train/docs_used": 548797,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.521967649459839,
+      "objective/train/theoretical_loss": 3.6551321784515434,
+      "objective/train/tokens_used": 1003500000,
+      "theoretical_loss": 3.6551321784515434,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003277851333459429,
+      "loss": 2.7408,
+      "theoretical_loss": 3.6551321784515434,
+      "tokens_seen": 983040000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003275487043692075,
+      "loss": 2.6923,
+      "theoretical_loss": 3.6546673088403985,
+      "tokens_seen": 984350720
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003273122753924721,
+      "loss": 2.69,
+      "theoretical_loss": 3.654203230875848,
+      "tokens_seen": 985661440
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032707584641573673,
+      "loss": 2.6432,
+      "theoretical_loss": 3.653739942159728,
+      "tokens_seen": 986972160
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003268394174390013,
+      "loss": 2.6929,
+      "theoretical_loss": 3.65327744030431,
+      "tokens_seen": 988282880
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003266029884622659,
+      "loss": 2.6817,
+      "theoretical_loss": 3.6528157229322447,
+      "tokens_seen": 989593600
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003263665594855306,
+      "loss": 2.7071,
+      "theoretical_loss": 3.6523547876765017,
+      "tokens_seen": 990904320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0003261301305087951,
+      "loss": 2.7686,
+      "theoretical_loss": 3.651894632180312,
+      "tokens_seen": 992215040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00032589370153205977,
+      "loss": 2.7383,
+      "theoretical_loss": 3.65143525409711,
+      "tokens_seen": 993525760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003256572725553244,
+      "loss": 2.7726,
+      "theoretical_loss": 3.650976651090474,
+      "tokens_seen": 994836480
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032542084357858897,
+      "loss": 2.8484,
+      "theoretical_loss": 3.650518820834072,
+      "tokens_seen": 996147200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003251844146018536,
+      "loss": 2.8759,
+      "theoretical_loss": 3.650061761011603,
+      "tokens_seen": 997457920
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003249479856251182,
+      "loss": 2.8348,
+      "theoretical_loss": 3.649605469316742,
+      "tokens_seen": 998768640
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003247115566483828,
+      "loss": 2.8273,
+      "theoretical_loss": 3.6491499434530814,
+      "tokens_seen": 1000079360
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032447512767164746,
+      "loss": 2.9085,
+      "theoretical_loss": 3.648695181134078,
+      "tokens_seen": 1001390080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032423869869491206,
+      "loss": 2.8267,
+      "theoretical_loss": 3.648241180082998,
+      "tokens_seen": 1002700800
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032400226971817666,
+      "loss": 2.8412,
+      "theoretical_loss": 3.64778793803286,
+      "tokens_seen": 1004011520
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032376584074144126,
+      "loss": 2.8551,
+      "theoretical_loss": 3.6473354527263813,
+      "tokens_seen": 1005322240
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003235294117647059,
+      "loss": 2.8016,
+      "theoretical_loss": 3.6468837219159256,
+      "tokens_seen": 1006632960
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003232929827879705,
+      "loss": 2.8096,
+      "theoretical_loss": 3.6464327433634467,
+      "tokens_seen": 1007943680
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003230565538112351,
+      "loss": 2.8058,
+      "theoretical_loss": 3.645982514840437,
+      "tokens_seen": 1009254400
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032282012483449975,
+      "loss": 2.8067,
+      "theoretical_loss": 3.6455330341278733,
+      "tokens_seen": 1010565120
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003225836958577643,
+      "loss": 2.7943,
+      "theoretical_loss": 3.645084299016166,
+      "tokens_seen": 1011875840
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032234726688102895,
+      "loss": 2.8197,
+      "theoretical_loss": 3.644636307305104,
+      "tokens_seen": 1013186560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032211083790429355,
+      "loss": 2.8741,
+      "theoretical_loss": 3.644189056803805,
+      "tokens_seen": 1014497280
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00032187440892755814,
+      "loss": 2.7665,
+      "theoretical_loss": 3.6437425453306638,
+      "tokens_seen": 1015808000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003216379799508228,
+      "loss": 2.7972,
+      "theoretical_loss": 3.643296770713301,
+      "tokens_seen": 1017118720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003214015509740874,
+      "loss": 2.8233,
+      "theoretical_loss": 3.642851730788511,
+      "tokens_seen": 1018429440
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.000321165121997352,
+      "loss": 2.812,
+      "theoretical_loss": 3.642407423402213,
+      "tokens_seen": 1019740160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0003209286930206166,
+      "loss": 2.8244,
+      "theoretical_loss": 3.641963846409399,
+      "tokens_seen": 1021050880
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032069226404388124,
+      "loss": 2.8051,
+      "theoretical_loss": 3.641520997674087,
+      "tokens_seen": 1022361600
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032045583506714584,
+      "loss": 2.7976,
+      "theoretical_loss": 3.641078875069269,
+      "tokens_seen": 1023672320
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00032021940609041044,
+      "loss": 2.7887,
+      "theoretical_loss": 3.640637476476862,
+      "tokens_seen": 1024983040
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003199829771136751,
+      "loss": 2.7397,
+      "theoretical_loss": 3.64019679978766,
+      "tokens_seen": 1026293760
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031974654813693963,
+      "loss": 2.7836,
+      "theoretical_loss": 3.639756842901286,
+      "tokens_seen": 1027604480
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003195101191602043,
+      "loss": 2.8272,
+      "theoretical_loss": 3.639317603726142,
+      "tokens_seen": 1028915200
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031927369018346893,
+      "loss": 2.7717,
+      "theoretical_loss": 3.638879080179364,
+      "tokens_seen": 1030225920
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003190372612067335,
+      "loss": 2.7526,
+      "theoretical_loss": 3.63844127018677,
+      "tokens_seen": 1031536640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031880083222999813,
+      "loss": 2.7991,
+      "theoretical_loss": 3.6380041716828186,
+      "tokens_seen": 1032847360
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003185644032532627,
+      "loss": 2.767,
+      "theoretical_loss": 3.637567782610558,
+      "tokens_seen": 1034158080
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003183279742765273,
+      "loss": 2.7628,
+      "theoretical_loss": 3.637132100921579,
+      "tokens_seen": 1035468800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000318091545299792,
+      "loss": 2.7679,
+      "theoretical_loss": 3.6366971245759734,
+      "tokens_seen": 1036779520
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031785511632305657,
+      "loss": 2.7715,
+      "theoretical_loss": 3.6362628515422815,
+      "tokens_seen": 1038090240
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031761868734632117,
+      "loss": 2.7894,
+      "theoretical_loss": 3.635829279797453,
+      "tokens_seen": 1039400960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031738225836958577,
+      "loss": 2.7991,
+      "theoretical_loss": 3.6353964073267964,
+      "tokens_seen": 1040711680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003171458293928504,
+      "loss": 2.8076,
+      "theoretical_loss": 3.6349642321239375,
+      "tokens_seen": 1042022400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.000316909400416115,
+      "loss": 2.7328,
+      "theoretical_loss": 3.6345327521907724,
+      "tokens_seen": 1043333120
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003166729714393796,
+      "loss": 2.8094,
+      "theoretical_loss": 3.6341019655374245,
+      "tokens_seen": 1044643840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031643654246264427,
+      "loss": 2.8167,
+      "theoretical_loss": 3.6336718701821997,
+      "tokens_seen": 1045954560
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0003162001134859088,
+      "loss": 2.8122,
+      "theoretical_loss": 3.633242464151543,
+      "tokens_seen": 1047265280
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 582102,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.794548511505127,
+      "objective/train/theoretical_loss": 3.632813745479994,
+      "objective/train/tokens_used": 1069036000,
+      "theoretical_loss": 3.632813745479994,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031596368450917346,
+      "loss": 2.7677,
+      "theoretical_loss": 3.632813745479994,
+      "tokens_seen": 1048576000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00031572725553243806,
+      "loss": 2.7831,
+      "theoretical_loss": 3.6323857122101444,
+      "tokens_seen": 1049886720
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031549082655570266,
+      "loss": 2.7946,
+      "theoretical_loss": 3.631958362392595,
+      "tokens_seen": 1051197440
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003152543975789673,
+      "loss": 2.7869,
+      "theoretical_loss": 3.631531694085914,
+      "tokens_seen": 1052508160
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003150179686022319,
+      "loss": 2.8095,
+      "theoretical_loss": 3.6311057053565907,
+      "tokens_seen": 1053818880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003147815396254965,
+      "loss": 2.8055,
+      "theoretical_loss": 3.6306803942789987,
+      "tokens_seen": 1055129600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003145451106487611,
+      "loss": 2.7616,
+      "theoretical_loss": 3.63025575893535,
+      "tokens_seen": 1056440320
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031430868167202575,
+      "loss": 2.7888,
+      "theoretical_loss": 3.6298317974156555,
+      "tokens_seen": 1057751040
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031407225269529035,
+      "loss": 2.7503,
+      "theoretical_loss": 3.629408507817682,
+      "tokens_seen": 1059061760
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031383582371855495,
+      "loss": 2.7281,
+      "theoretical_loss": 3.6289858882469135,
+      "tokens_seen": 1060372480
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003135993947418196,
+      "loss": 2.7001,
+      "theoretical_loss": 3.6285639368165077,
+      "tokens_seen": 1061683200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031336296576508414,
+      "loss": 2.7484,
+      "theoretical_loss": 3.6281426516472584,
+      "tokens_seen": 1062993920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003131265367883488,
+      "loss": 2.7319,
+      "theoretical_loss": 3.627722030867552,
+      "tokens_seen": 1064304640
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031289010781161345,
+      "loss": 2.7823,
+      "theoretical_loss": 3.62730207261333,
+      "tokens_seen": 1065615360
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.000312653678834878,
+      "loss": 2.7401,
+      "theoretical_loss": 3.626882775028048,
+      "tokens_seen": 1066926080
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031241724985814264,
+      "loss": 2.7448,
+      "theoretical_loss": 3.6264641362626375,
+      "tokens_seen": 1068236800
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031218082088140724,
+      "loss": 2.7133,
+      "theoretical_loss": 3.626046154475465,
+      "tokens_seen": 1069547520
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031194439190467183,
+      "loss": 2.7846,
+      "theoretical_loss": 3.625628827832294,
+      "tokens_seen": 1070858240
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003117079629279365,
+      "loss": 2.7132,
+      "theoretical_loss": 3.625212154506246,
+      "tokens_seen": 1072168960
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003114715339512011,
+      "loss": 2.7165,
+      "theoretical_loss": 3.624796132677763,
+      "tokens_seen": 1073479680
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003112351049744657,
+      "loss": 2.6974,
+      "theoretical_loss": 3.624380760534567,
+      "tokens_seen": 1074790400
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.0003109986759977303,
+      "loss": 2.672,
+      "theoretical_loss": 3.6239660362716246,
+      "tokens_seen": 1076101120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00031076224702099493,
+      "loss": 2.7422,
+      "theoretical_loss": 3.623551958091108,
+      "tokens_seen": 1077411840
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003105258180442595,
+      "loss": 2.7931,
+      "theoretical_loss": 3.623138524202358,
+      "tokens_seen": 1078722560
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003102893890675241,
+      "loss": 2.7098,
+      "theoretical_loss": 3.622725732821844,
+      "tokens_seen": 1080033280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003100529600907888,
+      "loss": 2.7654,
+      "theoretical_loss": 3.622313582173133,
+      "tokens_seen": 1081344000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003098165311140533,
+      "loss": 2.7464,
+      "theoretical_loss": 3.621902070486846,
+      "tokens_seen": 1082654720
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030958010213731797,
+      "loss": 2.7268,
+      "theoretical_loss": 3.6214911960006275,
+      "tokens_seen": 1083965440
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030934367316058257,
+      "loss": 2.7577,
+      "theoretical_loss": 3.621080956959104,
+      "tokens_seen": 1085276160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030910724418384717,
+      "loss": 2.7286,
+      "theoretical_loss": 3.6206713516138507,
+      "tokens_seen": 1086586880
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003088708152071118,
+      "loss": 2.6756,
+      "theoretical_loss": 3.620262378223357,
+      "tokens_seen": 1087897600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030863438623037636,
+      "loss": 2.7719,
+      "theoretical_loss": 3.619854035052987,
+      "tokens_seen": 1089208320
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.000308397957253641,
+      "loss": 2.7851,
+      "theoretical_loss": 3.619446320374947,
+      "tokens_seen": 1090519040
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003081615282769056,
+      "loss": 2.7505,
+      "theoretical_loss": 3.61903923246825,
+      "tokens_seen": 1091829760
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003079250993001702,
+      "loss": 2.7442,
+      "theoretical_loss": 3.6186327696186797,
+      "tokens_seen": 1093140480
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030768867032343486,
+      "loss": 2.757,
+      "theoretical_loss": 3.618226930118757,
+      "tokens_seen": 1094451200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030745224134669946,
+      "loss": 2.7201,
+      "theoretical_loss": 3.6178217122677045,
+      "tokens_seen": 1095761920
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030721581236996406,
+      "loss": 2.7424,
+      "theoretical_loss": 3.6174171143714124,
+      "tokens_seen": 1097072640
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030697938339322865,
+      "loss": 2.6363,
+      "theoretical_loss": 3.617013134742405,
+      "tokens_seen": 1098383360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003067429544164933,
+      "loss": 2.7293,
+      "theoretical_loss": 3.616609771699806,
+      "tokens_seen": 1099694080
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003065065254397579,
+      "loss": 2.7025,
+      "theoretical_loss": 3.616207023569306,
+      "tokens_seen": 1101004800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003062700964630225,
+      "loss": 2.6649,
+      "theoretical_loss": 3.6158048886831287,
+      "tokens_seen": 1102315520
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00030603366748628715,
+      "loss": 2.6875,
+      "theoretical_loss": 3.6154033653799953,
+      "tokens_seen": 1103626240
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0003057972385095517,
+      "loss": 2.7062,
+      "theoretical_loss": 3.615002452005096,
+      "tokens_seen": 1104936960
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030556080953281635,
+      "loss": 2.6877,
+      "theoretical_loss": 3.614602146910054,
+      "tokens_seen": 1106247680
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000305324380556081,
+      "loss": 2.744,
+      "theoretical_loss": 3.614202448452893,
+      "tokens_seen": 1107558400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030508795157934554,
+      "loss": 2.6732,
+      "theoretical_loss": 3.613803354998007,
+      "tokens_seen": 1108869120
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003048515226026102,
+      "loss": 2.6624,
+      "theoretical_loss": 3.6134048649161237,
+      "tokens_seen": 1110179840
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003046150936258748,
+      "loss": 2.7612,
+      "theoretical_loss": 3.6130069765842787,
+      "tokens_seen": 1111490560
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003043786646491394,
+      "loss": 2.7307,
+      "theoretical_loss": 3.6126096883857777,
+      "tokens_seen": 1112801280
+    },
+    {
+      "epoch": 0.4,
+      "objective/train/docs_used": 617270,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7352099418640137,
+      "objective/train/theoretical_loss": 3.61221299871017,
+      "objective/train/tokens_used": 1134572000,
+      "theoretical_loss": 3.61221299871017,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000304142235672404,
+      "loss": 2.7201,
+      "theoretical_loss": 3.61221299871017,
+      "tokens_seen": 1114112000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030390580669566864,
+      "loss": 2.7404,
+      "theoretical_loss": 3.611816905953212,
+      "tokens_seen": 1115422720
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030366937771893323,
+      "loss": 2.7276,
+      "theoretical_loss": 3.6114214085168417,
+      "tokens_seen": 1116733440
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030343294874219783,
+      "loss": 2.7574,
+      "theoretical_loss": 3.611026504809142,
+      "tokens_seen": 1118044160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003031965197654625,
+      "loss": 2.7338,
+      "theoretical_loss": 3.6106321932443137,
+      "tokens_seen": 1119354880
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.000302960090788727,
+      "loss": 2.6852,
+      "theoretical_loss": 3.6102384722426435,
+      "tokens_seen": 1120665600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003027236618119917,
+      "loss": 2.7177,
+      "theoretical_loss": 3.609845340230474,
+      "tokens_seen": 1121976320
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030248723283525633,
+      "loss": 2.7638,
+      "theoretical_loss": 3.6094527956401734,
+      "tokens_seen": 1123287040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003022508038585209,
+      "loss": 2.7358,
+      "theoretical_loss": 3.6090608369101043,
+      "tokens_seen": 1124597760
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003020143748817855,
+      "loss": 2.7572,
+      "theoretical_loss": 3.6086694624845954,
+      "tokens_seen": 1125908480
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003017779459050501,
+      "loss": 2.7159,
+      "theoretical_loss": 3.6082786708139114,
+      "tokens_seen": 1127219200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.0003015415169283147,
+      "loss": 2.732,
+      "theoretical_loss": 3.6078884603542223,
+      "tokens_seen": 1128529920
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030130508795157937,
+      "loss": 2.6587,
+      "theoretical_loss": 3.607498829567576,
+      "tokens_seen": 1129840640
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030106865897484397,
+      "loss": 2.7247,
+      "theoretical_loss": 3.6071097769218676,
+      "tokens_seen": 1131151360
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030083222999810857,
+      "loss": 2.7124,
+      "theoretical_loss": 3.606721300890811,
+      "tokens_seen": 1132462080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00030059580102137316,
+      "loss": 2.7556,
+      "theoretical_loss": 3.6063333999539102,
+      "tokens_seen": 1133772800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003003593720446378,
+      "loss": 2.7092,
+      "theoretical_loss": 3.6059460725964305,
+      "tokens_seen": 1135083520
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0003001229430679024,
+      "loss": 2.7517,
+      "theoretical_loss": 3.6055593173093694,
+      "tokens_seen": 1136394240
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000299886514091167,
+      "loss": 2.7249,
+      "theoretical_loss": 3.60517313258943,
+      "tokens_seen": 1137704960
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029965008511443166,
+      "loss": 2.6931,
+      "theoretical_loss": 3.604787516938992,
+      "tokens_seen": 1139015680
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002994136561376962,
+      "loss": 2.682,
+      "theoretical_loss": 3.6044024688660823,
+      "tokens_seen": 1140326400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029917722716096086,
+      "loss": 2.658,
+      "theoretical_loss": 3.604017986884349,
+      "tokens_seen": 1141637120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029894079818422545,
+      "loss": 2.7155,
+      "theoretical_loss": 3.603634069513035,
+      "tokens_seen": 1142947840
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029870436920749005,
+      "loss": 2.6954,
+      "theoretical_loss": 3.603250715276947,
+      "tokens_seen": 1144258560
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002984679402307547,
+      "loss": 2.6614,
+      "theoretical_loss": 3.6028679227064306,
+      "tokens_seen": 1145569280
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002982315112540193,
+      "loss": 2.7523,
+      "theoretical_loss": 3.6024856903373434,
+      "tokens_seen": 1146880000
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002979950822772839,
+      "loss": 2.7453,
+      "theoretical_loss": 3.602104016711027,
+      "tokens_seen": 1148190720
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002977586533005485,
+      "loss": 2.7876,
+      "theoretical_loss": 3.6017229003742806,
+      "tokens_seen": 1149501440
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029752222432381315,
+      "loss": 2.7734,
+      "theoretical_loss": 3.6013423398793347,
+      "tokens_seen": 1150812160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029728579534707775,
+      "loss": 2.7786,
+      "theoretical_loss": 3.600962333783824,
+      "tokens_seen": 1152122880
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029704936637034234,
+      "loss": 2.748,
+      "theoretical_loss": 3.6005828806507623,
+      "tokens_seen": 1153433600
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.000296812937393607,
+      "loss": 2.7681,
+      "theoretical_loss": 3.6002039790485147,
+      "tokens_seen": 1154744320
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029657650841687154,
+      "loss": 2.724,
+      "theoretical_loss": 3.5998256275507723,
+      "tokens_seen": 1156055040
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002963400794401362,
+      "loss": 2.7365,
+      "theoretical_loss": 3.5994478247365276,
+      "tokens_seen": 1157365760
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029610365046340084,
+      "loss": 2.7361,
+      "theoretical_loss": 3.5990705691900464,
+      "tokens_seen": 1158676480
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0002958672214866654,
+      "loss": 2.7441,
+      "theoretical_loss": 3.5986938595008455,
+      "tokens_seen": 1159987200
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00029563079250993004,
+      "loss": 2.7271,
+      "theoretical_loss": 3.5983176942636637,
+      "tokens_seen": 1161297920
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029539436353319463,
+      "loss": 2.6954,
+      "theoretical_loss": 3.597942072078439,
+      "tokens_seen": 1162608640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029515793455645923,
+      "loss": 2.6957,
+      "theoretical_loss": 3.5975669915502833,
+      "tokens_seen": 1163919360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002949215055797239,
+      "loss": 2.6952,
+      "theoretical_loss": 3.597192451289457,
+      "tokens_seen": 1165230080
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002946850766029885,
+      "loss": 2.7206,
+      "theoretical_loss": 3.5968184499113436,
+      "tokens_seen": 1166540800
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002944486476262531,
+      "loss": 2.7274,
+      "theoretical_loss": 3.5964449860364263,
+      "tokens_seen": 1167851520
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002942122186495177,
+      "loss": 2.7106,
+      "theoretical_loss": 3.5960720582902628,
+      "tokens_seen": 1169162240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029397578967278233,
+      "loss": 2.722,
+      "theoretical_loss": 3.595699665303462,
+      "tokens_seen": 1170472960
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002937393606960469,
+      "loss": 2.6853,
+      "theoretical_loss": 3.595327805711657,
+      "tokens_seen": 1171783680
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002935029317193115,
+      "loss": 2.7167,
+      "theoretical_loss": 3.5949564781554857,
+      "tokens_seen": 1173094400
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002932665027425762,
+      "loss": 2.7573,
+      "theoretical_loss": 3.5945856812805626,
+      "tokens_seen": 1174405120
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002930300737658407,
+      "loss": 2.7663,
+      "theoretical_loss": 3.594215413737457,
+      "tokens_seen": 1175715840
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029279364478910537,
+      "loss": 2.6982,
+      "theoretical_loss": 3.5938456741816682,
+      "tokens_seen": 1177026560
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029255721581236997,
+      "loss": 2.7246,
+      "theoretical_loss": 3.593476461273605,
+      "tokens_seen": 1178337280
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 651196,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.479915142059326,
+      "objective/train/theoretical_loss": 3.5931077736785584,
+      "objective/train/tokens_used": 1200108000,
+      "theoretical_loss": 3.5931077736785584,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029232078683563456,
+      "loss": 2.726,
+      "theoretical_loss": 3.5931077736785584,
+      "tokens_seen": 1179648000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002920843578588992,
+      "loss": 2.7187,
+      "theoretical_loss": 3.5927396100666806,
+      "tokens_seen": 1180958720
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002918479288821638,
+      "loss": 2.7085,
+      "theoretical_loss": 3.592371969112962,
+      "tokens_seen": 1182269440
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.0002916114999054284,
+      "loss": 2.7093,
+      "theoretical_loss": 3.5920048494972074,
+      "tokens_seen": 1183580160
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.000291375070928693,
+      "loss": 2.7153,
+      "theoretical_loss": 3.5916382499040136,
+      "tokens_seen": 1184890880
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029113864195195766,
+      "loss": 2.6854,
+      "theoretical_loss": 3.591272169022747,
+      "tokens_seen": 1186201600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029090221297522226,
+      "loss": 2.7137,
+      "theoretical_loss": 3.590906605547521,
+      "tokens_seen": 1187512320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00029066578399848685,
+      "loss": 2.7606,
+      "theoretical_loss": 3.590541558177172,
+      "tokens_seen": 1188823040
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002904293550217515,
+      "loss": 2.7493,
+      "theoretical_loss": 3.5901770256152403,
+      "tokens_seen": 1190133760
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00029019292604501605,
+      "loss": 2.7247,
+      "theoretical_loss": 3.589813006569945,
+      "tokens_seen": 1191444480
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002899564970682807,
+      "loss": 2.715,
+      "theoretical_loss": 3.589449499754164,
+      "tokens_seen": 1192755200
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028972006809154535,
+      "loss": 2.7259,
+      "theoretical_loss": 3.5890865038854103,
+      "tokens_seen": 1194065920
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002894836391148099,
+      "loss": 2.6994,
+      "theoretical_loss": 3.5887240176858115,
+      "tokens_seen": 1195376640
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028924721013807455,
+      "loss": 2.6993,
+      "theoretical_loss": 3.588362039882088,
+      "tokens_seen": 1196687360
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028901078116133915,
+      "loss": 2.7149,
+      "theoretical_loss": 3.5880005692055317,
+      "tokens_seen": 1197998080
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028877435218460374,
+      "loss": 2.6354,
+      "theoretical_loss": 3.5876396043919834,
+      "tokens_seen": 1199308800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002885379232078684,
+      "loss": 2.711,
+      "theoretical_loss": 3.587279144181813,
+      "tokens_seen": 1200619520
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.000288301494231133,
+      "loss": 2.7516,
+      "theoretical_loss": 3.586919187319897,
+      "tokens_seen": 1201930240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002880650652543976,
+      "loss": 2.7652,
+      "theoretical_loss": 3.5865597325555987,
+      "tokens_seen": 1203240960
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002878286362776622,
+      "loss": 2.7374,
+      "theoretical_loss": 3.5862007786427474,
+      "tokens_seen": 1204551680
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028759220730092684,
+      "loss": 2.7198,
+      "theoretical_loss": 3.585842324339616,
+      "tokens_seen": 1205862400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002873557783241914,
+      "loss": 2.6784,
+      "theoretical_loss": 3.5854843684089017,
+      "tokens_seen": 1207173120
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028711934934745603,
+      "loss": 2.7206,
+      "theoretical_loss": 3.5851269096177054,
+      "tokens_seen": 1208483840
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002868829203707207,
+      "loss": 2.713,
+      "theoretical_loss": 3.5847699467375103,
+      "tokens_seen": 1209794560
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00028664649139398523,
+      "loss": 2.7114,
+      "theoretical_loss": 3.584413478544163,
+      "tokens_seen": 1211105280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002864100624172499,
+      "loss": 2.7686,
+      "theoretical_loss": 3.584057503817852,
+      "tokens_seen": 1212416000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002861736334405145,
+      "loss": 2.7292,
+      "theoretical_loss": 3.5837020213430884,
+      "tokens_seen": 1213726720
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002859372044637791,
+      "loss": 2.7163,
+      "theoretical_loss": 3.583347029908686,
+      "tokens_seen": 1215037440
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002857007754870437,
+      "loss": 2.7902,
+      "theoretical_loss": 3.582992528307741,
+      "tokens_seen": 1216348160
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0002854643465103083,
+      "loss": 2.7219,
+      "theoretical_loss": 3.5826385153376124,
+      "tokens_seen": 1217658880
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002852279175335729,
+      "loss": 2.6844,
+      "theoretical_loss": 3.582284989799902,
+      "tokens_seen": 1218969600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002849914885568375,
+      "loss": 2.7532,
+      "theoretical_loss": 3.5819319505004366,
+      "tokens_seen": 1220280320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002847550595801021,
+      "loss": 2.7505,
+      "theoretical_loss": 3.581579396249245,
+      "tokens_seen": 1221591040
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028451863060336677,
+      "loss": 2.7297,
+      "theoretical_loss": 3.5812273258605436,
+      "tokens_seen": 1222901760
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028428220162663137,
+      "loss": 2.7273,
+      "theoretical_loss": 3.580875738152713,
+      "tokens_seen": 1224212480
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028404577264989596,
+      "loss": 2.7384,
+      "theoretical_loss": 3.5805246319482817,
+      "tokens_seen": 1225523200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028380934367316056,
+      "loss": 2.7338,
+      "theoretical_loss": 3.5801740060739036,
+      "tokens_seen": 1226833920
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002835729146964252,
+      "loss": 2.7568,
+      "theoretical_loss": 3.579823859360345,
+      "tokens_seen": 1228144640
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002833364857196898,
+      "loss": 2.7162,
+      "theoretical_loss": 3.5794741906424594,
+      "tokens_seen": 1229455360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002831000567429544,
+      "loss": 2.7057,
+      "theoretical_loss": 3.5791249987591733,
+      "tokens_seen": 1230766080
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028286362776621906,
+      "loss": 2.6861,
+      "theoretical_loss": 3.5787762825534664,
+      "tokens_seen": 1232076800
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002826271987894836,
+      "loss": 2.7547,
+      "theoretical_loss": 3.578428040872351,
+      "tokens_seen": 1233387520
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028239076981274825,
+      "loss": 2.731,
+      "theoretical_loss": 3.5780802725668575,
+      "tokens_seen": 1234698240
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002821543408360129,
+      "loss": 2.7291,
+      "theoretical_loss": 3.5777329764920136,
+      "tokens_seen": 1236008960
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028191791185927745,
+      "loss": 2.7434,
+      "theoretical_loss": 3.5773861515068273,
+      "tokens_seen": 1237319680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002816814828825421,
+      "loss": 2.768,
+      "theoretical_loss": 3.5770397964742675,
+      "tokens_seen": 1238630400
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002814450539058067,
+      "loss": 2.7136,
+      "theoretical_loss": 3.5766939102612474,
+      "tokens_seen": 1239941120
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002812086249290713,
+      "loss": 2.7569,
+      "theoretical_loss": 3.5763484917386057,
+      "tokens_seen": 1241251840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0002809721959523359,
+      "loss": 2.7926,
+      "theoretical_loss": 3.576003539781091,
+      "tokens_seen": 1242562560
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028073576697560054,
+      "loss": 2.7553,
+      "theoretical_loss": 3.5756590532673407,
+      "tokens_seen": 1243873280
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 687081,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.7497341632843018,
+      "objective/train/theoretical_loss": 3.5753150310798665,
+      "objective/train/tokens_used": 1265644000,
+      "theoretical_loss": 3.5753150310798665,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00028049933799886514,
+      "loss": 2.7684,
+      "theoretical_loss": 3.5753150310798665,
+      "tokens_seen": 1245184000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00028026290902212974,
+      "loss": 2.7927,
+      "theoretical_loss": 3.5749714721050343,
+      "tokens_seen": 1246494720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002800264800453944,
+      "loss": 2.7293,
+      "theoretical_loss": 3.5746283752330505,
+      "tokens_seen": 1247805440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027979005106865893,
+      "loss": 2.7781,
+      "theoretical_loss": 3.574285739357941,
+      "tokens_seen": 1249116160
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002795536220919236,
+      "loss": 2.7713,
+      "theoretical_loss": 3.5739435633775365,
+      "tokens_seen": 1250426880
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027931719311518824,
+      "loss": 2.7508,
+      "theoretical_loss": 3.573601846193453,
+      "tokens_seen": 1251737600
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002790807641384528,
+      "loss": 2.792,
+      "theoretical_loss": 3.573260586711079,
+      "tokens_seen": 1253048320
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027884433516171743,
+      "loss": 2.7227,
+      "theoretical_loss": 3.5729197838395543,
+      "tokens_seen": 1254359040
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027860790618498203,
+      "loss": 2.7278,
+      "theoretical_loss": 3.5725794364917554,
+      "tokens_seen": 1255669760
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027837147720824663,
+      "loss": 2.7971,
+      "theoretical_loss": 3.5722395435842778,
+      "tokens_seen": 1256980480
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002781350482315113,
+      "loss": 2.7059,
+      "theoretical_loss": 3.5719001040374216,
+      "tokens_seen": 1258291200
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002778986192547759,
+      "loss": 2.7023,
+      "theoretical_loss": 3.5715611167751717,
+      "tokens_seen": 1259601920
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002776621902780405,
+      "loss": 2.7328,
+      "theoretical_loss": 3.571222580725184,
+      "tokens_seen": 1260912640
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027742576130130507,
+      "loss": 2.689,
+      "theoretical_loss": 3.5708844948187686,
+      "tokens_seen": 1262223360
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002771893323245697,
+      "loss": 2.7101,
+      "theoretical_loss": 3.5705468579908715,
+      "tokens_seen": 1263534080
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002769529033478343,
+      "loss": 2.6872,
+      "theoretical_loss": 3.5702096691800627,
+      "tokens_seen": 1264844800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002767164743710989,
+      "loss": 2.6923,
+      "theoretical_loss": 3.5698729273285146,
+      "tokens_seen": 1266155520
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027648004539436357,
+      "loss": 2.7046,
+      "theoretical_loss": 3.5695366313819914,
+      "tokens_seen": 1267466240
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.0002762436164176281,
+      "loss": 2.7169,
+      "theoretical_loss": 3.56920078028983,
+      "tokens_seen": 1268776960
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027600718744089277,
+      "loss": 2.735,
+      "theoretical_loss": 3.5688653730049245,
+      "tokens_seen": 1270087680
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027577075846415736,
+      "loss": 2.721,
+      "theoretical_loss": 3.5685304084837126,
+      "tokens_seen": 1271398400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00027553432948742196,
+      "loss": 2.727,
+      "theoretical_loss": 3.5681958856861558,
+      "tokens_seen": 1272709120
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002752979005106866,
+      "loss": 2.6792,
+      "theoretical_loss": 3.5678618035757297,
+      "tokens_seen": 1274019840
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002750614715339512,
+      "loss": 2.708,
+      "theoretical_loss": 3.5675281611194025,
+      "tokens_seen": 1275330560
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002748250425572158,
+      "loss": 2.7403,
+      "theoretical_loss": 3.5671949572876245,
+      "tokens_seen": 1276641280
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002745886135804804,
+      "loss": 2.7104,
+      "theoretical_loss": 3.5668621910543097,
+      "tokens_seen": 1277952000
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027435218460374506,
+      "loss": 2.6361,
+      "theoretical_loss": 3.566529861396822,
+      "tokens_seen": 1279262720
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027411575562700965,
+      "loss": 2.6962,
+      "theoretical_loss": 3.566197967295959,
+      "tokens_seen": 1280573440
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027387932665027425,
+      "loss": 2.6861,
+      "theoretical_loss": 3.56586650773594,
+      "tokens_seen": 1281884160
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027365471912237565,
+      "loss": 2.725,
+      "theoretical_loss": 3.5655354817043863,
+      "tokens_seen": 1283194880
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027341829014564025,
+      "loss": 2.6541,
+      "theoretical_loss": 3.5652048881923095,
+      "tokens_seen": 1284505600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027318186116890485,
+      "loss": 2.6734,
+      "theoretical_loss": 3.5648747261940974,
+      "tokens_seen": 1285816320
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002729454321921695,
+      "loss": 2.6738,
+      "theoretical_loss": 3.5645449947074956,
+      "tokens_seen": 1287127040
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027270900321543404,
+      "loss": 2.6183,
+      "theoretical_loss": 3.5642156927335975,
+      "tokens_seen": 1288437760
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002724725742386987,
+      "loss": 2.6836,
+      "theoretical_loss": 3.563886819276826,
+      "tokens_seen": 1289748480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027223614526196335,
+      "loss": 2.7058,
+      "theoretical_loss": 3.5635583733449203,
+      "tokens_seen": 1291059200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002719997162852279,
+      "loss": 2.6559,
+      "theoretical_loss": 3.5632303539489225,
+      "tokens_seen": 1292369920
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027176328730849254,
+      "loss": 2.6548,
+      "theoretical_loss": 3.562902760103161,
+      "tokens_seen": 1293680640
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027152685833175714,
+      "loss": 2.672,
+      "theoretical_loss": 3.56257559082524,
+      "tokens_seen": 1294991360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00027129042935502174,
+      "loss": 2.7554,
+      "theoretical_loss": 3.562248845136021,
+      "tokens_seen": 1296302080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002710540003782864,
+      "loss": 2.6672,
+      "theoretical_loss": 3.5619225220596116,
+      "tokens_seen": 1297612800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.000270817571401551,
+      "loss": 2.6859,
+      "theoretical_loss": 3.56159662062335,
+      "tokens_seen": 1298923520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002705811424248156,
+      "loss": 2.6827,
+      "theoretical_loss": 3.5612711398577916,
+      "tokens_seen": 1300234240
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0002703447134480802,
+      "loss": 2.6652,
+      "theoretical_loss": 3.5609460787966962,
+      "tokens_seen": 1301544960
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00027010828447134483,
+      "loss": 2.6778,
+      "theoretical_loss": 3.560621436477012,
+      "tokens_seen": 1302855680
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026987185549460943,
+      "loss": 2.6807,
+      "theoretical_loss": 3.560297211938863,
+      "tokens_seen": 1304166400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026963542651787403,
+      "loss": 2.6414,
+      "theoretical_loss": 3.5599734042255373,
+      "tokens_seen": 1305477120
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002693989975411387,
+      "loss": 2.7098,
+      "theoretical_loss": 3.559650012383469,
+      "tokens_seen": 1306787840
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002691625685644032,
+      "loss": 2.6631,
+      "theoretical_loss": 3.559327035462228,
+      "tokens_seen": 1308098560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002689261395876679,
+      "loss": 2.6442,
+      "theoretical_loss": 3.5590044725145074,
+      "tokens_seen": 1309409280
+    },
+    {
+      "debugging/Self-BLEU-5": 0.4426207415415405,
+      "debugging/distinct-1-grams": 0.7890487734679326,
+      "debugging/distinct-2-grams": 0.9586932642491364,
+      "debugging/entropy-1-grams": 5.5999886701178365,
+      "debugging/entropy-2-grams": 6.248945838762987,
+      "debugging/length": 492.8888888888889,
+      "debugging/num_segments": 9,
+      "epoch": 0.47,
+      "objective/train/docs_used": 722210,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.6524410247802734,
+      "objective/train/theoretical_loss": 3.5586823225961064,
+      "objective/train/tokens_used": 1331180000,
+      "theoretical_loss": 3.5586823225961064,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002686897106109325,
+      "loss": 2.6465,
+      "theoretical_loss": 3.5586823225961064,
+      "tokens_seen": 1310720000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026845328163419707,
+      "loss": 2.6664,
+      "theoretical_loss": 3.5583605847659214,
+      "tokens_seen": 1312030720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002682168526574617,
+      "loss": 2.5936,
+      "theoretical_loss": 3.558039258085929,
+      "tokens_seen": 1313341440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002679804236807263,
+      "loss": 2.6648,
+      "theoretical_loss": 3.5577183416211744,
+      "tokens_seen": 1314652160
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002677439947039909,
+      "loss": 2.6637,
+      "theoretical_loss": 3.5573978344397594,
+      "tokens_seen": 1315962880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026750756572725557,
+      "loss": 2.6402,
+      "theoretical_loss": 3.5570777356128276,
+      "tokens_seen": 1317273600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026727113675052016,
+      "loss": 2.6487,
+      "theoretical_loss": 3.556758044214552,
+      "tokens_seen": 1318584320
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026703470777378476,
+      "loss": 2.6394,
+      "theoretical_loss": 3.5564387593221225,
+      "tokens_seen": 1319895040
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026679827879704936,
+      "loss": 2.6472,
+      "theoretical_loss": 3.556119880015733,
+      "tokens_seen": 1321205760
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.000266561849820314,
+      "loss": 2.6149,
+      "theoretical_loss": 3.5558014053785687,
+      "tokens_seen": 1322516480
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026632542084357855,
+      "loss": 2.7127,
+      "theoretical_loss": 3.5554833344967927,
+      "tokens_seen": 1323827200
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002660889918668432,
+      "loss": 2.6169,
+      "theoretical_loss": 3.5551656664595344,
+      "tokens_seen": 1325137920
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026585256289010786,
+      "loss": 2.6687,
+      "theoretical_loss": 3.5548484003588756,
+      "tokens_seen": 1326448640
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0002656161339133724,
+      "loss": 2.6844,
+      "theoretical_loss": 3.554531535289841,
+      "tokens_seen": 1327759360
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00026537970493663705,
+      "loss": 2.6875,
+      "theoretical_loss": 3.5542150703503808,
+      "tokens_seen": 1329070080
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026514327595990165,
+      "loss": 2.6809,
+      "theoretical_loss": 3.553899004641363,
+      "tokens_seen": 1330380800
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026490684698316625,
+      "loss": 2.694,
+      "theoretical_loss": 3.55358333726656,
+      "tokens_seen": 1331691520
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002646704180064309,
+      "loss": 2.6529,
+      "theoretical_loss": 3.553268067332634,
+      "tokens_seen": 1333002240
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002644339890296955,
+      "loss": 2.6668,
+      "theoretical_loss": 3.5529531939491275,
+      "tokens_seen": 1334312960
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002641975600529601,
+      "loss": 2.7029,
+      "theoretical_loss": 3.552638716228449,
+      "tokens_seen": 1335623680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002639611310762247,
+      "loss": 2.7027,
+      "theoretical_loss": 3.552324633285865,
+      "tokens_seen": 1336934400
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026372470209948934,
+      "loss": 2.6892,
+      "theoretical_loss": 3.5520109442394827,
+      "tokens_seen": 1338245120
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026348827312275394,
+      "loss": 2.6785,
+      "theoretical_loss": 3.5516976482102414,
+      "tokens_seen": 1339555840
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026325184414601854,
+      "loss": 2.7096,
+      "theoretical_loss": 3.5513847443218998,
+      "tokens_seen": 1340866560
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002630154151692832,
+      "loss": 2.6688,
+      "theoretical_loss": 3.551072231701025,
+      "tokens_seen": 1342177280
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026277898619254773,
+      "loss": 2.6756,
+      "theoretical_loss": 3.550760109476979,
+      "tokens_seen": 1343488000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002625425572158124,
+      "loss": 2.7086,
+      "theoretical_loss": 3.550448376781909,
+      "tokens_seen": 1344798720
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026230612823907704,
+      "loss": 2.6703,
+      "theoretical_loss": 3.550137032750734,
+      "tokens_seen": 1346109440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002620696992623416,
+      "loss": 2.6597,
+      "theoretical_loss": 3.5498260765211356,
+      "tokens_seen": 1347420160
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026183327028560623,
+      "loss": 2.687,
+      "theoretical_loss": 3.549515507233543,
+      "tokens_seen": 1348730880
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026159684130887083,
+      "loss": 2.7065,
+      "theoretical_loss": 3.5492053240311257,
+      "tokens_seen": 1350041600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002613604123321354,
+      "loss": 2.6078,
+      "theoretical_loss": 3.5488955260597783,
+      "tokens_seen": 1351352320
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002611239833554001,
+      "loss": 2.7119,
+      "theoretical_loss": 3.5485861124681124,
+      "tokens_seen": 1352663040
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002608875543786647,
+      "loss": 2.6809,
+      "theoretical_loss": 3.5482770824074428,
+      "tokens_seen": 1353973760
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.0002606511254019293,
+      "loss": 2.7116,
+      "theoretical_loss": 3.5479684350317777,
+      "tokens_seen": 1355284480
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026041469642519387,
+      "loss": 2.7642,
+      "theoretical_loss": 3.5476601694978083,
+      "tokens_seen": 1356595200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00026017826744845847,
+      "loss": 2.6952,
+      "theoretical_loss": 3.5473522849648944,
+      "tokens_seen": 1357905920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025994183847172307,
+      "loss": 2.651,
+      "theoretical_loss": 3.547044780595058,
+      "tokens_seen": 1359216640
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002597054094949877,
+      "loss": 2.6915,
+      "theoretical_loss": 3.5467376555529686,
+      "tokens_seen": 1360527360
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002594689805182523,
+      "loss": 2.6372,
+      "theoretical_loss": 3.546430909005934,
+      "tokens_seen": 1361838080
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002592325515415169,
+      "loss": 2.6183,
+      "theoretical_loss": 3.5461245401238894,
+      "tokens_seen": 1363148800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025899612256478156,
+      "loss": 2.7269,
+      "theoretical_loss": 3.545818548079385,
+      "tokens_seen": 1364459520
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002587596935880461,
+      "loss": 2.6855,
+      "theoretical_loss": 3.545512932047579,
+      "tokens_seen": 1365770240
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025852326461131076,
+      "loss": 2.6475,
+      "theoretical_loss": 3.5452076912062216,
+      "tokens_seen": 1367080960
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002582868356345754,
+      "loss": 2.6807,
+      "theoretical_loss": 3.54490282473565,
+      "tokens_seen": 1368391680
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025805040665783995,
+      "loss": 2.6939,
+      "theoretical_loss": 3.544598331818772,
+      "tokens_seen": 1369702400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002578139776811046,
+      "loss": 2.7045,
+      "theoretical_loss": 3.5442942116410614,
+      "tokens_seen": 1371013120
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002575775487043692,
+      "loss": 2.6584,
+      "theoretical_loss": 3.5439904633905424,
+      "tokens_seen": 1372323840
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002573411197276338,
+      "loss": 2.6989,
+      "theoretical_loss": 3.5436870862577825,
+      "tokens_seen": 1373634560
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025710469075089845,
+      "loss": 2.7251,
+      "theoretical_loss": 3.5433840794358806,
+      "tokens_seen": 1374945280
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 759831,
+      "objective/train/instantaneous_batch_size": 32,
+      "objective/train/instantaneous_microbatch_size": 32768,
+      "objective/train/original_loss": 2.380300283432007,
+      "objective/train/theoretical_loss": 3.5430814421204575,
+      "objective/train/tokens_used": 1396716000,
+      "theoretical_loss": 3.5430814421204575,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025686826177416305,
+      "loss": 2.6341,
+      "theoretical_loss": 3.5430814421204575,
+      "tokens_seen": 1376256000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025663183279742765,
+      "loss": 2.6611,
+      "theoretical_loss": 3.5427791735096448,
+      "tokens_seen": 1377566720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025639540382069225,
+      "loss": 2.6876,
+      "theoretical_loss": 3.542477272804075,
+      "tokens_seen": 1378877440
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002561589748439569,
+      "loss": 2.6203,
+      "theoretical_loss": 3.542175739206872,
+      "tokens_seen": 1380188160
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002559225458672215,
+      "loss": 2.6141,
+      "theoretical_loss": 3.541874571923639,
+      "tokens_seen": 1381498880
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002556861168904861,
+      "loss": 2.6862,
+      "theoretical_loss": 3.5415737701624517,
+      "tokens_seen": 1382809600
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00025544968791375074,
+      "loss": 2.6507,
+      "theoretical_loss": 3.5412733331338453,
+      "tokens_seen": 1384120320
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0002552250803858521,
+      "loss": 2.6561,
+      "theoretical_loss": 3.5409732600508055,
+      "tokens_seen": 1385431040
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002549886514091167,
+      "loss": 2.6833,
+      "theoretical_loss": 3.5406735501287594,
+      "tokens_seen": 1386741760
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025475222243238134,
+      "loss": 2.6558,
+      "theoretical_loss": 3.5403742025855647,
+      "tokens_seen": 1388052480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002545157934556459,
+      "loss": 2.6706,
+      "theoretical_loss": 3.5400752166415,
+      "tokens_seen": 1389363200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025427936447891054,
+      "loss": 2.6581,
+      "theoretical_loss": 3.5397765915192547,
+      "tokens_seen": 1390673920
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002540429355021752,
+      "loss": 2.7117,
+      "theoretical_loss": 3.5394783264439207,
+      "tokens_seen": 1391984640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025380650652543973,
+      "loss": 2.6816,
+      "theoretical_loss": 3.5391804206429818,
+      "tokens_seen": 1393295360
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002535700775487044,
+      "loss": 2.6913,
+      "theoretical_loss": 3.538882873346303,
+      "tokens_seen": 1394606080
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.000253333648571969,
+      "loss": 2.7268,
+      "theoretical_loss": 3.5385856837861223,
+      "tokens_seen": 1395916800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002530972195952336,
+      "loss": 2.7259,
+      "theoretical_loss": 3.538288851197042,
+      "tokens_seen": 1397227520
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00025286079061849823,
+      "loss": 2.6803,
+      "theoretical_loss": 3.537992374816016,
+      "tokens_seen": 1398538240
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.0002526243616417628,
+      "loss": 2.6769,
+      "theoretical_loss": 3.537696253882345,
+      "tokens_seen": 1399848960
+    }
+  ],
+  "max_steps": 42724,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 7.14460209610752e+17,
+  "trial_name": null,
+  "trial_params": null
+}