diff --git "a/checkpoint-12588/trainer_state.json" "b/checkpoint-12588/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-12588/trainer_state.json"
@@ -0,0 +1,90907 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "global_step": 12588,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507937e-07,
+      "loss": 2.9672,
+      "theoretical_loss": 3.515363695025882,
+      "tokens_seen": 1650130944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015873e-06,
+      "loss": 3.0022,
+      "theoretical_loss": 3.515339720665981,
+      "tokens_seen": 1650262016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.3809523809523808e-06,
+      "loss": 2.9482,
+      "theoretical_loss": 3.5153157487432845,
+      "tokens_seen": 1650393088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031746e-06,
+      "loss": 2.8923,
+      "theoretical_loss": 3.5152917792573506,
+      "tokens_seen": 1650524160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-06,
+      "loss": 2.8362,
+      "theoretical_loss": 3.515267812207739,
+      "tokens_seen": 1650655232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.7619047619047615e-06,
+      "loss": 2.9705,
+      "theoretical_loss": 3.515243847594008,
+      "tokens_seen": 1650786304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.555555555555556e-06,
+      "loss": 2.9071,
+      "theoretical_loss": 3.515219885415717,
+      "tokens_seen": 1650917376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.349206349206349e-06,
+      "loss": 2.831,
+      "theoretical_loss": 3.5151959256724252,
+      "tokens_seen": 1651048448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.142857142857143e-06,
+      "loss": 2.7254,
+      "theoretical_loss": 3.515171968363691,
+      "tokens_seen": 1651179520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.936507936507936e-06,
+      "loss": 2.7267,
+      "theoretical_loss": 3.5151480134890747,
+      "tokens_seen": 1651310592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.73015873015873e-06,
+      "loss": 2.6015,
+      "theoretical_loss": 3.5151240610481356,
+      "tokens_seen": 1651441664
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.523809523809523e-06,
+      "loss": 2.6527,
+      "theoretical_loss": 3.5151001110404336,
+      "tokens_seen": 1651572736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.0317460317460318e-05,
+      "loss": 2.6256,
+      "theoretical_loss": 3.515076163465528,
+      "tokens_seen": 1651703808
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 2.5568,
+      "theoretical_loss": 3.515052218322979,
+      "tokens_seen": 1651834880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1904761904761905e-05,
+      "loss": 2.5464,
+      "theoretical_loss": 3.515028275612346,
+      "tokens_seen": 1651965952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2698412698412699e-05,
+      "loss": 2.5071,
+      "theoretical_loss": 3.5150043353331895,
+      "tokens_seen": 1652097024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3492063492063492e-05,
+      "loss": 2.4226,
+      "theoretical_loss": 3.5149803974850697,
+      "tokens_seen": 1652228096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 2.3967,
+      "theoretical_loss": 3.5149564620675466,
+      "tokens_seen": 1652359168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5079365079365079e-05,
+      "loss": 2.4456,
+      "theoretical_loss": 3.5149325290801814,
+      "tokens_seen": 1652490240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.5873015873015872e-05,
+      "loss": 2.4331,
+      "theoretical_loss": 3.514908598522534,
+      "tokens_seen": 1652621312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 2.3243,
+      "theoretical_loss": 3.5148846703941654,
+      "tokens_seen": 1652752384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.746031746031746e-05,
+      "loss": 2.2979,
+      "theoretical_loss": 3.5148607446946354,
+      "tokens_seen": 1652883456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.8253968253968254e-05,
+      "loss": 2.2525,
+      "theoretical_loss": 3.514836821423506,
+      "tokens_seen": 1653014528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9047619047619046e-05,
+      "loss": 2.1948,
+      "theoretical_loss": 3.5148129005803384,
+      "tokens_seen": 1653145600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.984126984126984e-05,
+      "loss": 2.3082,
+      "theoretical_loss": 3.5147889821646925,
+      "tokens_seen": 1653276672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.0634920634920636e-05,
+      "loss": 2.1541,
+      "theoretical_loss": 3.51476506617613,
+      "tokens_seen": 1653407744
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.1428571428571428e-05,
+      "loss": 2.2103,
+      "theoretical_loss": 3.514741152614213,
+      "tokens_seen": 1653538816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2222222222222223e-05,
+      "loss": 2.1856,
+      "theoretical_loss": 3.514717241478502,
+      "tokens_seen": 1653669888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.3015873015873015e-05,
+      "loss": 2.2149,
+      "theoretical_loss": 3.5146933327685597,
+      "tokens_seen": 1653800960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.380952380952381e-05,
+      "loss": 2.1401,
+      "theoretical_loss": 3.5146694264839464,
+      "tokens_seen": 1653932032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.4603174603174602e-05,
+      "loss": 2.1379,
+      "theoretical_loss": 3.5146455226242246,
+      "tokens_seen": 1654063104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.5396825396825397e-05,
+      "loss": 2.0204,
+      "theoretical_loss": 3.514621621188956,
+      "tokens_seen": 1654194176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6190476190476192e-05,
+      "loss": 2.1312,
+      "theoretical_loss": 3.5145977221777027,
+      "tokens_seen": 1654325248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6984126984126984e-05,
+      "loss": 1.9966,
+      "theoretical_loss": 3.514573825590027,
+      "tokens_seen": 1654456320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.777777777777778e-05,
+      "loss": 2.0464,
+      "theoretical_loss": 3.514549931425491,
+      "tokens_seen": 1654587392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 2.0983,
+      "theoretical_loss": 3.5145260396836573,
+      "tokens_seen": 1654718464
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.9365079365079366e-05,
+      "loss": 2.0193,
+      "theoretical_loss": 3.514502150364088,
+      "tokens_seen": 1654849536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.0158730158730158e-05,
+      "loss": 2.0713,
+      "theoretical_loss": 3.514478263466346,
+      "tokens_seen": 1654980608
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.095238095238095e-05,
+      "loss": 1.9273,
+      "theoretical_loss": 3.514454378989994,
+      "tokens_seen": 1655111680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1746031746031745e-05,
+      "loss": 2.0133,
+      "theoretical_loss": 3.5144304969345947,
+      "tokens_seen": 1655242752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.253968253968254e-05,
+      "loss": 1.8852,
+      "theoretical_loss": 3.5144066172997106,
+      "tokens_seen": 1655373824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.9259,
+      "theoretical_loss": 3.5143827400849057,
+      "tokens_seen": 1655504896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.412698412698413e-05,
+      "loss": 1.9136,
+      "theoretical_loss": 3.514358865289742,
+      "tokens_seen": 1655635968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.492063492063492e-05,
+      "loss": 1.794,
+      "theoretical_loss": 3.514334992913784,
+      "tokens_seen": 1655767040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.571428571428572e-05,
+      "loss": 1.8864,
+      "theoretical_loss": 3.5143111229565944,
+      "tokens_seen": 1655898112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.650793650793651e-05,
+      "loss": 1.8455,
+      "theoretical_loss": 3.514287255417737,
+      "tokens_seen": 1656029184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.730158730158731e-05,
+      "loss": 1.7438,
+      "theoretical_loss": 3.514263390296775,
+      "tokens_seen": 1656160256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.809523809523809e-05,
+      "loss": 1.7546,
+      "theoretical_loss": 3.5142395275932725,
+      "tokens_seen": 1656291328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.888888888888889e-05,
+      "loss": 1.768,
+      "theoretical_loss": 3.514215667306793,
+      "tokens_seen": 1656422400
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/docs_used": 557587,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.7760109901428223,
+      "objective/train/theoretical_loss": 3.514191809436901,
+      "objective/train/tokens_used": 27012576,
+      "theoretical_loss": 3.514191809436901,
+      "tokens_seen": 1656553472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.968253968253968e-05,
+      "loss": 1.6981,
+      "theoretical_loss": 3.514191809436901,
+      "tokens_seen": 1656553472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.047619047619048e-05,
+      "loss": 1.6488,
+      "theoretical_loss": 3.5141679539831596,
+      "tokens_seen": 1656684544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.126984126984127e-05,
+      "loss": 1.6451,
+      "theoretical_loss": 3.514144100945134,
+      "tokens_seen": 1656815616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.2063492063492065e-05,
+      "loss": 1.6252,
+      "theoretical_loss": 3.514120250322388,
+      "tokens_seen": 1656946688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 1.582,
+      "theoretical_loss": 3.514096402114486,
+      "tokens_seen": 1657077760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.3650793650793655e-05,
+      "loss": 1.6163,
+      "theoretical_loss": 3.514072556320993,
+      "tokens_seen": 1657208832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 1.5095,
+      "theoretical_loss": 3.514048712941473,
+      "tokens_seen": 1657339904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.523809523809524e-05,
+      "loss": 1.5225,
+      "theoretical_loss": 3.514024871975491,
+      "tokens_seen": 1657470976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.603174603174603e-05,
+      "loss": 1.5782,
+      "theoretical_loss": 3.5140010334226117,
+      "tokens_seen": 1657602048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.682539682539683e-05,
+      "loss": 1.5169,
+      "theoretical_loss": 3.5139771972824008,
+      "tokens_seen": 1657733120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 1.4823,
+      "theoretical_loss": 3.513953363554422,
+      "tokens_seen": 1657864192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.841269841269841e-05,
+      "loss": 1.5075,
+      "theoretical_loss": 3.513929532238242,
+      "tokens_seen": 1657995264
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.9206349206349204e-05,
+      "loss": 1.4983,
+      "theoretical_loss": 3.5139057033334247,
+      "tokens_seen": 1658126336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5e-05,
+      "loss": 1.4664,
+      "theoretical_loss": 3.5138818768395366,
+      "tokens_seen": 1658257408
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.0793650793650794e-05,
+      "loss": 1.4523,
+      "theoretical_loss": 3.513858052756143,
+      "tokens_seen": 1658388480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.158730158730159e-05,
+      "loss": 1.4878,
+      "theoretical_loss": 3.513834231082809,
+      "tokens_seen": 1658519552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.2380952380952384e-05,
+      "loss": 1.5102,
+      "theoretical_loss": 3.5138104118191014,
+      "tokens_seen": 1658650624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.3174603174603176e-05,
+      "loss": 1.4698,
+      "theoretical_loss": 3.513786594964585,
+      "tokens_seen": 1658781696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.396825396825397e-05,
+      "loss": 1.4267,
+      "theoretical_loss": 3.5137627805188263,
+      "tokens_seen": 1658912768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.4761904761904766e-05,
+      "loss": 1.3926,
+      "theoretical_loss": 3.5137389684813916,
+      "tokens_seen": 1659043840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 1.3163,
+      "theoretical_loss": 3.5137151588518467,
+      "tokens_seen": 1659174912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.634920634920635e-05,
+      "loss": 1.4613,
+      "theoretical_loss": 3.513691351629758,
+      "tokens_seen": 1659305984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 1.4284,
+      "theoretical_loss": 3.5136675468146916,
+      "tokens_seen": 1659437056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.793650793650795e-05,
+      "loss": 1.3994,
+      "theoretical_loss": 3.513643744406215,
+      "tokens_seen": 1659568128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.873015873015873e-05,
+      "loss": 1.4238,
+      "theoretical_loss": 3.513619944403894,
+      "tokens_seen": 1659699200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.9523809523809524e-05,
+      "loss": 1.43,
+      "theoretical_loss": 3.513596146807296,
+      "tokens_seen": 1659830272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.0317460317460316e-05,
+      "loss": 1.3626,
+      "theoretical_loss": 3.513572351615987,
+      "tokens_seen": 1659961344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.111111111111112e-05,
+      "loss": 1.3841,
+      "theoretical_loss": 3.5135485588295348,
+      "tokens_seen": 1660092416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.19047619047619e-05,
+      "loss": 1.3375,
+      "theoretical_loss": 3.513524768447507,
+      "tokens_seen": 1660223488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.26984126984127e-05,
+      "loss": 1.3236,
+      "theoretical_loss": 3.513500980469469,
+      "tokens_seen": 1660354560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.349206349206349e-05,
+      "loss": 1.4346,
+      "theoretical_loss": 3.51347719489499,
+      "tokens_seen": 1660485632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.428571428571429e-05,
+      "loss": 1.3466,
+      "theoretical_loss": 3.513453411723636,
+      "tokens_seen": 1660616704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.507936507936509e-05,
+      "loss": 1.4196,
+      "theoretical_loss": 3.5134296309549757,
+      "tokens_seen": 1660747776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.587301587301587e-05,
+      "loss": 1.2848,
+      "theoretical_loss": 3.513405852588576,
+      "tokens_seen": 1660878848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.513382076624005,
+      "tokens_seen": 1661009920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.746031746031747e-05,
+      "loss": 1.3212,
+      "theoretical_loss": 3.5133583030608304,
+      "tokens_seen": 1661140992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.825396825396825e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.5133345318986207,
+      "tokens_seen": 1661272064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.904761904761905e-05,
+      "loss": 1.3413,
+      "theoretical_loss": 3.5133107631369436,
+      "tokens_seen": 1661403136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.984126984126984e-05,
+      "loss": 1.3083,
+      "theoretical_loss": 3.513286996775367,
+      "tokens_seen": 1661534208
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.063492063492065e-05,
+      "loss": 1.3121,
+      "theoretical_loss": 3.5132632328134603,
+      "tokens_seen": 1661665280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 1.3474,
+      "theoretical_loss": 3.513239471250791,
+      "tokens_seen": 1661796352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.222222222222222e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.513215712086928,
+      "tokens_seen": 1661927424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.301587301587302e-05,
+      "loss": 1.3343,
+      "theoretical_loss": 3.5131919553214397,
+      "tokens_seen": 1662058496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.380952380952382e-05,
+      "loss": 1.3342,
+      "theoretical_loss": 3.5131682009538947,
+      "tokens_seen": 1662189568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.460317460317461e-05,
+      "loss": 1.3193,
+      "theoretical_loss": 3.513144448983863,
+      "tokens_seen": 1662320640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.53968253968254e-05,
+      "loss": 1.3389,
+      "theoretical_loss": 3.513120699410913,
+      "tokens_seen": 1662451712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.513096952234613,
+      "tokens_seen": 1662582784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.6984126984127e-05,
+      "loss": 1.3386,
+      "theoretical_loss": 3.513073207454534,
+      "tokens_seen": 1662713856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.777777777777778e-05,
+      "loss": 1.3306,
+      "theoretical_loss": 3.5130494650702433,
+      "tokens_seen": 1662844928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.857142857142858e-05,
+      "loss": 1.3639,
+      "theoretical_loss": 3.513025725081312,
+      "tokens_seen": 1662976000
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 559635,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3444054126739502,
+      "objective/train/theoretical_loss": 3.513001987487309,
+      "objective/train/tokens_used": 33566176,
+      "theoretical_loss": 3.513001987487309,
+      "tokens_seen": 1663107072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.936507936507937e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.513001987487309,
+      "tokens_seen": 1663107072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.015873015873016e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.5129782522878035,
+      "tokens_seen": 1663238144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.095238095238096e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.512954519482366,
+      "tokens_seen": 1663369216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.174603174603175e-05,
+      "loss": 1.2959,
+      "theoretical_loss": 3.512930789070566,
+      "tokens_seen": 1663500288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.253968253968255e-05,
+      "loss": 1.2934,
+      "theoretical_loss": 3.5129070610519744,
+      "tokens_seen": 1663631360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 1.2956,
+      "theoretical_loss": 3.51288333542616,
+      "tokens_seen": 1663762432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.412698412698413e-05,
+      "loss": 1.3167,
+      "theoretical_loss": 3.5128596121926936,
+      "tokens_seen": 1663893504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.492063492063493e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.5128358913511457,
+      "tokens_seen": 1664024576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.512812172901087,
+      "tokens_seen": 1664155648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.650793650793651e-05,
+      "loss": 1.334,
+      "theoretical_loss": 3.5127884568420873,
+      "tokens_seen": 1664286720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.730158730158731e-05,
+      "loss": 1.3254,
+      "theoretical_loss": 3.512764743173718,
+      "tokens_seen": 1664417792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.80952380952381e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.5127410318955494,
+      "tokens_seen": 1664548864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.888888888888889e-05,
+      "loss": 1.3422,
+      "theoretical_loss": 3.512717323007153,
+      "tokens_seen": 1664679936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 8.968253968253969e-05,
+      "loss": 1.2651,
+      "theoretical_loss": 3.512693616508099,
+      "tokens_seen": 1664811008
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.047619047619048e-05,
+      "loss": 1.2869,
+      "theoretical_loss": 3.512669912397959,
+      "tokens_seen": 1664942080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.126984126984128e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.5126462106763037,
+      "tokens_seen": 1665073152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.206349206349206e-05,
+      "loss": 1.3369,
+      "theoretical_loss": 3.5126225113427054,
+      "tokens_seen": 1665204224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.285714285714286e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.5125988143967346,
+      "tokens_seen": 1665335296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.365079365079366e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.5125751198379636,
+      "tokens_seen": 1665466368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.444444444444444e-05,
+      "loss": 1.3187,
+      "theoretical_loss": 3.5125514276659637,
+      "tokens_seen": 1665597440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 1.368,
+      "theoretical_loss": 3.5125277378803066,
+      "tokens_seen": 1665728512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.603174603174604e-05,
+      "loss": 1.3132,
+      "theoretical_loss": 3.5125040504805645,
+      "tokens_seen": 1665859584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.682539682539682e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.5124803654663093,
+      "tokens_seen": 1665990656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.761904761904762e-05,
+      "loss": 1.3457,
+      "theoretical_loss": 3.512456682837113,
+      "tokens_seen": 1666121728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.841269841269841e-05,
+      "loss": 1.3076,
+      "theoretical_loss": 3.5124330025925476,
+      "tokens_seen": 1666252800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.920634920634922e-05,
+      "loss": 1.3746,
+      "theoretical_loss": 3.512409324732186,
+      "tokens_seen": 1666383872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0001,
+      "loss": 1.3491,
+      "theoretical_loss": 3.5123856492556005,
+      "tokens_seen": 1666514944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.999197560584176e-05,
+      "loss": 1.2924,
+      "theoretical_loss": 3.5123619761623632,
+      "tokens_seen": 1666646016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.998395121168352e-05,
+      "loss": 1.4305,
+      "theoretical_loss": 3.5123383054520474,
+      "tokens_seen": 1666777088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.997592681752528e-05,
+      "loss": 1.3463,
+      "theoretical_loss": 3.5123146371242253,
+      "tokens_seen": 1666908160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.996790242336704e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.51229097117847,
+      "tokens_seen": 1667039232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.99598780292088e-05,
+      "loss": 1.3588,
+      "theoretical_loss": 3.512267307614355,
+      "tokens_seen": 1667170304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.995185363505056e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.512243646431452,
+      "tokens_seen": 1667301376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.994382924089232e-05,
+      "loss": 1.4263,
+      "theoretical_loss": 3.512219987629336,
+      "tokens_seen": 1667432448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.993580484673407e-05,
+      "loss": 1.368,
+      "theoretical_loss": 3.512196331207579,
+      "tokens_seen": 1667563520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.992778045257584e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.5121726771657555,
+      "tokens_seen": 1667694592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.991975605841759e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.512149025503438,
+      "tokens_seen": 1667825664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.991173166425936e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.512125376220201,
+      "tokens_seen": 1667956736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.990370727010111e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.5121017293156176,
+      "tokens_seen": 1668087808
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.989568287594288e-05,
+      "loss": 1.3398,
+      "theoretical_loss": 3.512078084789262,
+      "tokens_seen": 1668218880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.988765848178463e-05,
+      "loss": 1.3179,
+      "theoretical_loss": 3.512054442640708,
+      "tokens_seen": 1668349952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.987963408762638e-05,
+      "loss": 1.2722,
+      "theoretical_loss": 3.51203080286953,
+      "tokens_seen": 1668481024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.987160969346815e-05,
+      "loss": 1.3426,
+      "theoretical_loss": 3.5120071654753025,
+      "tokens_seen": 1668612096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.98635852993099e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.511983530457599,
+      "tokens_seen": 1668743168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.985556090515167e-05,
+      "loss": 1.3306,
+      "theoretical_loss": 3.5119598978159945,
+      "tokens_seen": 1668874240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.984753651099342e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.5119362675500634,
+      "tokens_seen": 1669005312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.983951211683519e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.51191263965938,
+      "tokens_seen": 1669136384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.983148772267694e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.5118890141435193,
+      "tokens_seen": 1669267456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.982346332851871e-05,
+      "loss": 1.3934,
+      "theoretical_loss": 3.5118653910020563,
+      "tokens_seen": 1669398528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.981543893436046e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.511841770234566,
+      "tokens_seen": 1669529600
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/docs_used": 561964,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3444902896881104,
+      "objective/train/theoretical_loss": 3.5118181518406235,
+      "objective/train/tokens_used": 40119776,
+      "theoretical_loss": 3.5118181518406235,
+      "tokens_seen": 1669660672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.980741454020222e-05,
+      "loss": 1.3508,
+      "theoretical_loss": 3.5118181518406235,
+      "tokens_seen": 1669660672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.979939014604398e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.5117945358198037,
+      "tokens_seen": 1669791744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.979136575188574e-05,
+      "loss": 1.3437,
+      "theoretical_loss": 3.5117709221716815,
+      "tokens_seen": 1669922816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.97833413577275e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.5117473108958337,
+      "tokens_seen": 1670053888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.977531696356926e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.5117237019918344,
+      "tokens_seen": 1670184960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.976729256941102e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.5117000954592603,
+      "tokens_seen": 1670316032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.975926817525277e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.5116764912976866,
+      "tokens_seen": 1670447104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.975124378109453e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.5116528895066885,
+      "tokens_seen": 1670578176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.97432193869363e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.5116292900858435,
+      "tokens_seen": 1670709248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.973519499277805e-05,
+      "loss": 1.3366,
+      "theoretical_loss": 3.5116056930347264,
+      "tokens_seen": 1670840320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.972717059861981e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.511582098352914,
+      "tokens_seen": 1670971392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.971914620446157e-05,
+      "loss": 1.2925,
+      "theoretical_loss": 3.5115585060399823,
+      "tokens_seen": 1671102464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.971112181030333e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.5115349160955076,
+      "tokens_seen": 1671233536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.970309741614509e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.5115113285190667,
+      "tokens_seen": 1671364608
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.969507302198684e-05,
+      "loss": 1.3074,
+      "theoretical_loss": 3.5114877433102363,
+      "tokens_seen": 1671495680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.968704862782861e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.5114641604685928,
+      "tokens_seen": 1671626752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.967902423367036e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.511440579993713,
+      "tokens_seen": 1671757824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.967099983951213e-05,
+      "loss": 1.2962,
+      "theoretical_loss": 3.5114170018851736,
+      "tokens_seen": 1671888896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.966297544535388e-05,
+      "loss": 1.3385,
+      "theoretical_loss": 3.511393426142553,
+      "tokens_seen": 1672019968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.965495105119565e-05,
+      "loss": 1.3158,
+      "theoretical_loss": 3.5113698527654265,
+      "tokens_seen": 1672151040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.96469266570374e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.5113462817533727,
+      "tokens_seen": 1672282112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.963890226287915e-05,
+      "loss": 1.389,
+      "theoretical_loss": 3.5113227131059688,
+      "tokens_seen": 1672413184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.963087786872092e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.5112991468227914,
+      "tokens_seen": 1672544256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.962285347456267e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.511275582903419,
+      "tokens_seen": 1672675328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.961482908040444e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.5112520213474285,
+      "tokens_seen": 1672806400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.960680468624619e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.5112284621543983,
+      "tokens_seen": 1672937472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.959878029208796e-05,
+      "loss": 1.326,
+      "theoretical_loss": 3.5112049053239067,
+      "tokens_seen": 1673068544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.959075589792971e-05,
+      "loss": 1.4227,
+      "theoretical_loss": 3.511181350855531,
+      "tokens_seen": 1673199616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.958273150377147e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.5111577987488496,
+      "tokens_seen": 1673330688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.957470710961323e-05,
+      "loss": 1.3166,
+      "theoretical_loss": 3.5111342490034407,
+      "tokens_seen": 1673461760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.956668271545499e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.5111107016188825,
+      "tokens_seen": 1673592832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.955865832129675e-05,
+      "loss": 1.3277,
+      "theoretical_loss": 3.511087156594754,
+      "tokens_seen": 1673723904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.95506339271385e-05,
+      "loss": 1.3015,
+      "theoretical_loss": 3.5110636139306326,
+      "tokens_seen": 1673854976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.954260953298027e-05,
+      "loss": 1.3165,
+      "theoretical_loss": 3.5110400736260985,
+      "tokens_seen": 1673986048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.953458513882203e-05,
+      "loss": 1.3149,
+      "theoretical_loss": 3.5110165356807292,
+      "tokens_seen": 1674117120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.952656074466378e-05,
+      "loss": 1.3692,
+      "theoretical_loss": 3.5109930000941043,
+      "tokens_seen": 1674248192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.951853635050554e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.5109694668658027,
+      "tokens_seen": 1674379264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.95105119563473e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.5109459359954034,
+      "tokens_seen": 1674510336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.950248756218906e-05,
+      "loss": 1.3351,
+      "theoretical_loss": 3.510922407482486,
+      "tokens_seen": 1674641408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.949446316803082e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.5108988813266286,
+      "tokens_seen": 1674772480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.948643877387258e-05,
+      "loss": 1.3168,
+      "theoretical_loss": 3.510875357527412,
+      "tokens_seen": 1674903552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.947841437971434e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.510851836084415,
+      "tokens_seen": 1675034624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.94703899855561e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.510828316997218,
+      "tokens_seen": 1675165696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.946236559139786e-05,
+      "loss": 1.3241,
+      "theoretical_loss": 3.5108048002654,
+      "tokens_seen": 1675296768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.945434119723961e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.510781285888541,
+      "tokens_seen": 1675427840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.944631680308138e-05,
+      "loss": 1.2892,
+      "theoretical_loss": 3.510757773866221,
+      "tokens_seen": 1675558912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.943829240892313e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.51073426419802,
+      "tokens_seen": 1675689984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.94302680147649e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.5107107568835185,
+      "tokens_seen": 1675821056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.942224362060665e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.5106872519222962,
+      "tokens_seen": 1675952128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.941421922644842e-05,
+      "loss": 1.325,
+      "theoretical_loss": 3.5106637493139345,
+      "tokens_seen": 1676083200
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 564144,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.182065486907959,
+      "objective/train/theoretical_loss": 3.510640249058013,
+      "objective/train/tokens_used": 46673376,
+      "theoretical_loss": 3.510640249058013,
+      "tokens_seen": 1676214272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.940619483229017e-05,
+      "loss": 1.3156,
+      "theoretical_loss": 3.510640249058013,
+      "tokens_seen": 1676214272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.939817043813192e-05,
+      "loss": 1.3029,
+      "theoretical_loss": 3.510616751154113,
+      "tokens_seen": 1676345344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.939014604397369e-05,
+      "loss": 1.3176,
+      "theoretical_loss": 3.5105932556018145,
+      "tokens_seen": 1676476416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.938212164981544e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.510569762400699,
+      "tokens_seen": 1676607488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.937409725565721e-05,
+      "loss": 1.3236,
+      "theoretical_loss": 3.510546271550347,
+      "tokens_seen": 1676738560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.936607286149896e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.510522783050339,
+      "tokens_seen": 1676869632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.935804846734073e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.5104992969002575,
+      "tokens_seen": 1677000704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.935002407318248e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.5104758130996836,
+      "tokens_seen": 1677131776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.934199967902424e-05,
+      "loss": 1.3481,
+      "theoretical_loss": 3.5104523316481973,
+      "tokens_seen": 1677262848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.9333975284866e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.510428852545381,
+      "tokens_seen": 1677393920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.932595089070776e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.5104053757908167,
+      "tokens_seen": 1677524992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.931792649654952e-05,
+      "loss": 1.3036,
+      "theoretical_loss": 3.5103819013840853,
+      "tokens_seen": 1677656064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.930990210239128e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.5103584293247687,
+      "tokens_seen": 1677787136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.930187770823304e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.5103349596124493,
+      "tokens_seen": 1677918208
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.92938533140748e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.510311492246709,
+      "tokens_seen": 1678049280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.928582891991655e-05,
+      "loss": 1.3292,
+      "theoretical_loss": 3.5102880272271295,
+      "tokens_seen": 1678180352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.927780452575832e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.5102645645532933,
+      "tokens_seen": 1678311424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.926978013160007e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.5102411042247823,
+      "tokens_seen": 1678442496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.926175573744183e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.51021764624118,
+      "tokens_seen": 1678573568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.925373134328359e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.5101941906020677,
+      "tokens_seen": 1678704640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.924570694912535e-05,
+      "loss": 1.3573,
+      "theoretical_loss": 3.510170737307029,
+      "tokens_seen": 1678835712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.923768255496711e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.510147286355646,
+      "tokens_seen": 1678966784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.922965816080886e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.5101238377475026,
+      "tokens_seen": 1679097856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.922163376665063e-05,
+      "loss": 1.31,
+      "theoretical_loss": 3.5101003914821804,
+      "tokens_seen": 1679228928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.921360937249238e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.510076947559263,
+      "tokens_seen": 1679360000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.920558497833415e-05,
+      "loss": 1.3167,
+      "theoretical_loss": 3.510053505978334,
+      "tokens_seen": 1679491072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.91975605841759e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.510030066738976,
+      "tokens_seen": 1679622144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.918953619001767e-05,
+      "loss": 1.2904,
+      "theoretical_loss": 3.510006629840773,
+      "tokens_seen": 1679753216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.918151179585942e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.5099831952833087,
+      "tokens_seen": 1679884288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.917348740170119e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.5099597630661656,
+      "tokens_seen": 1680015360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.916546300754294e-05,
+      "loss": 1.3192,
+      "theoretical_loss": 3.5099363331889286,
+      "tokens_seen": 1680146432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.91574386133847e-05,
+      "loss": 1.1975,
+      "theoretical_loss": 3.5099129056511806,
+      "tokens_seen": 1680277504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.914941421922646e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.5098894804525065,
+      "tokens_seen": 1680408576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.914138982506821e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.5098660575924896,
+      "tokens_seen": 1680539648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.913336543090998e-05,
+      "loss": 1.3415,
+      "theoretical_loss": 3.5098426370707143,
+      "tokens_seen": 1680670720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.912534103675173e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.509819218886765,
+      "tokens_seen": 1680801792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.91173166425935e-05,
+      "loss": 1.3223,
+      "theoretical_loss": 3.509795803040226,
+      "tokens_seen": 1680932864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.910929224843525e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.5097723895306814,
+      "tokens_seen": 1681063936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.9101267854277e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.509748978357716,
+      "tokens_seen": 1681195008
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.909324346011877e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.5097255695209144,
+      "tokens_seen": 1681326080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.908521906596053e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.5097021630198615,
+      "tokens_seen": 1681457152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.907719467180229e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.5096787588541423,
+      "tokens_seen": 1681588224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.906917027764405e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.509655357023342,
+      "tokens_seen": 1681719296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.906114588348581e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.509631957527045,
+      "tokens_seen": 1681850368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.905312148932757e-05,
+      "loss": 1.2945,
+      "theoretical_loss": 3.5096085603648364,
+      "tokens_seen": 1681981440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.904509709516932e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.509585165536303,
+      "tokens_seen": 1682112512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.903707270101109e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.5095617730410282,
+      "tokens_seen": 1682243584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.902904830685284e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.509538382878599,
+      "tokens_seen": 1682374656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.90210239126946e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.509514995048601,
+      "tokens_seen": 1682505728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.901299951853636e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.5094916095506195,
+      "tokens_seen": 1682636800
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 566429,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2210923433303833,
+      "objective/train/theoretical_loss": 3.5094682263842394,
+      "objective/train/tokens_used": 53226976,
+      "theoretical_loss": 3.5094682263842394,
+      "tokens_seen": 1682767872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.900497512437812e-05,
+      "loss": 1.2841,
+      "theoretical_loss": 3.5094682263842394,
+      "tokens_seen": 1682767872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.899695073021988e-05,
+      "loss": 1.3337,
+      "theoretical_loss": 3.509444845549049,
+      "tokens_seen": 1682898944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.898892633606163e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.5094214670446315,
+      "tokens_seen": 1683030016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.89809019419034e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.509398090870575,
+      "tokens_seen": 1683161088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.897287754774515e-05,
+      "loss": 1.2643,
+      "theoretical_loss": 3.509374717026466,
+      "tokens_seen": 1683292160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.896485315358692e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.509351345511889,
+      "tokens_seen": 1683423232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.895682875942867e-05,
+      "loss": 1.3021,
+      "theoretical_loss": 3.509327976326433,
+      "tokens_seen": 1683554304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.894880436527044e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.509304609469682,
+      "tokens_seen": 1683685376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.894077997111219e-05,
+      "loss": 1.3146,
+      "theoretical_loss": 3.5092812449412243,
+      "tokens_seen": 1683816448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.893275557695394e-05,
+      "loss": 1.2907,
+      "theoretical_loss": 3.5092578827406467,
+      "tokens_seen": 1683947520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.892473118279571e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.5092345228675352,
+      "tokens_seen": 1684078592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.891670678863746e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.5092111653214775,
+      "tokens_seen": 1684209664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.890868239447923e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.509187810102061,
+      "tokens_seen": 1684340736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.890065800032098e-05,
+      "loss": 1.3572,
+      "theoretical_loss": 3.509164457208872,
+      "tokens_seen": 1684471808
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.889263360616275e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.509141106641498,
+      "tokens_seen": 1684602880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.88846092120045e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.5091177583995266,
+      "tokens_seen": 1684733952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.887658481784626e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.5090944124825456,
+      "tokens_seen": 1684865024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.886856042368802e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.509071068890143,
+      "tokens_seen": 1684996096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.886053602952978e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.509047727621905,
+      "tokens_seen": 1685127168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.885251163537154e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.509024388677421,
+      "tokens_seen": 1685258240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.88444872412133e-05,
+      "loss": 1.1232,
+      "theoretical_loss": 3.509001052056279,
+      "tokens_seen": 1685389312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.883646284705506e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.508977717758065,
+      "tokens_seen": 1685520384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.882843845289682e-05,
+      "loss": 1.2438,
+      "theoretical_loss": 3.5089543857823697,
+      "tokens_seen": 1685651456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.882041405873857e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.5089310561287803,
+      "tokens_seen": 1685782528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.881238966458034e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.508907728796885,
+      "tokens_seen": 1685913600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.880436527042209e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.5088844037862725,
+      "tokens_seen": 1686044672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.879634087626386e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.508861081096531,
+      "tokens_seen": 1686175744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.878831648210561e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.50883776072725,
+      "tokens_seen": 1686306816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.878029208794736e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.508814442678017,
+      "tokens_seen": 1686437888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.877226769378913e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.508791126948423,
+      "tokens_seen": 1686568960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.876424329963088e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.5087678135380544,
+      "tokens_seen": 1686700032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.875621890547265e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.5087445024465023,
+      "tokens_seen": 1686831104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.87481945113144e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.508721193673355,
+      "tokens_seen": 1686962176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.874017011715617e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.5086978872182017,
+      "tokens_seen": 1687093248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.873214572299792e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.5086745830806327,
+      "tokens_seen": 1687224320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.872412132883967e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.508651281260237,
+      "tokens_seen": 1687355392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.871609693468144e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.5086279817566037,
+      "tokens_seen": 1687486464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.87080725405232e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.508604684569324,
+      "tokens_seen": 1687617536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.870004814636496e-05,
+      "loss": 1.3132,
+      "theoretical_loss": 3.508581389697986,
+      "tokens_seen": 1687748608
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.869202375220671e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.5085580971421804,
+      "tokens_seen": 1687879680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.868399935804847e-05,
+      "loss": 1.3211,
+      "theoretical_loss": 3.5085348069014977,
+      "tokens_seen": 1688010752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.867597496389023e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.5085115189755274,
+      "tokens_seen": 1688141824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.866795056973199e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.5084882333638596,
+      "tokens_seen": 1688272896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.865992617557375e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.508464950066086,
+      "tokens_seen": 1688403968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.86519017814155e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.5084416690817957,
+      "tokens_seen": 1688535040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.864387738725727e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.5084183904105792,
+      "tokens_seen": 1688666112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.863585299309903e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.5083951140520284,
+      "tokens_seen": 1688797184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.862782859894078e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.508371840005733,
+      "tokens_seen": 1688928256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.861980420478255e-05,
+      "loss": 1.2688,
+      "theoretical_loss": 3.508348568271284,
+      "tokens_seen": 1689059328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.86117798106243e-05,
+      "loss": 1.1846,
+      "theoretical_loss": 3.508325298848273,
+      "tokens_seen": 1689190400
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/docs_used": 568731,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1827294826507568,
+      "objective/train/theoretical_loss": 3.5083020317362905,
+      "objective/train/tokens_used": 59780576,
+      "theoretical_loss": 3.5083020317362905,
+      "tokens_seen": 1689321472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.860375541646607e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.5083020317362905,
+      "tokens_seen": 1689321472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.859573102230782e-05,
+      "loss": 1.2102,
+      "theoretical_loss": 3.508278766934928,
+      "tokens_seen": 1689452544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.858770662814957e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.5082555044437767,
+      "tokens_seen": 1689583616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.857968223399134e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.5082322442624276,
+      "tokens_seen": 1689714688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.857165783983309e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.508208986390473,
+      "tokens_seen": 1689845760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.856363344567486e-05,
+      "loss": 1.1726,
+      "theoretical_loss": 3.5081857308275044,
+      "tokens_seen": 1689976832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.855560905151661e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.5081624775731126,
+      "tokens_seen": 1690107904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.854758465735838e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.5081392266268905,
+      "tokens_seen": 1690238976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.853956026320013e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.508115977988429,
+      "tokens_seen": 1690370048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.853153586904188e-05,
+      "loss": 1.2645,
+      "theoretical_loss": 3.5080927316573214,
+      "tokens_seen": 1690501120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.852351147488365e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.508069487633159,
+      "tokens_seen": 1690632192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.85154870807254e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.508046245915534,
+      "tokens_seen": 1690763264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.850746268656717e-05,
+      "loss": 1.1784,
+      "theoretical_loss": 3.5080230065040388,
+      "tokens_seen": 1690894336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.849943829240892e-05,
+      "loss": 1.2586,
+      "theoretical_loss": 3.5079997693982663,
+      "tokens_seen": 1691025408
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.849141389825068e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.507976534597808,
+      "tokens_seen": 1691156480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.848338950409244e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.507953302102258,
+      "tokens_seen": 1691287552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.84753651099342e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.507930071911208,
+      "tokens_seen": 1691418624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.846734071577596e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.507906844024251,
+      "tokens_seen": 1691549696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.845931632161772e-05,
+      "loss": 1.1973,
+      "theoretical_loss": 3.5078836184409803,
+      "tokens_seen": 1691680768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.845129192745948e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.507860395160989,
+      "tokens_seen": 1691811840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.844326753330124e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.5078371741838694,
+      "tokens_seen": 1691942912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.843524313914299e-05,
+      "loss": 1.1277,
+      "theoretical_loss": 3.507813955509216,
+      "tokens_seen": 1692073984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.842721874498476e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.507790739136621,
+      "tokens_seen": 1692205056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.841919435082651e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.507767525065679,
+      "tokens_seen": 1692336128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.841116995666828e-05,
+      "loss": 1.165,
+      "theoretical_loss": 3.5077443132959827,
+      "tokens_seen": 1692467200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.840314556251003e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.5077211038271257,
+      "tokens_seen": 1692598272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.839512116835178e-05,
+      "loss": 1.3333,
+      "theoretical_loss": 3.507697896658703,
+      "tokens_seen": 1692729344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.838709677419355e-05,
+      "loss": 1.3301,
+      "theoretical_loss": 3.5076746917903074,
+      "tokens_seen": 1692860416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.83790723800353e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.5076514892215327,
+      "tokens_seen": 1692991488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.837104798587707e-05,
+      "loss": 1.3157,
+      "theoretical_loss": 3.507628288951974,
+      "tokens_seen": 1693122560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.836302359171882e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.507605090981225,
+      "tokens_seen": 1693253632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.835499919756059e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.5075818953088795,
+      "tokens_seen": 1693384704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.834697480340234e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.507558701934533,
+      "tokens_seen": 1693515776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.83389504092441e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.507535510857779,
+      "tokens_seen": 1693646848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.833092601508586e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.5075123220782123,
+      "tokens_seen": 1693777920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.832290162092762e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.507489135595428,
+      "tokens_seen": 1693908992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.831487722676938e-05,
+      "loss": 1.1981,
+      "theoretical_loss": 3.5074659514090207,
+      "tokens_seen": 1694040064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.830685283261113e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.5074427695185855,
+      "tokens_seen": 1694171136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.82988284384529e-05,
+      "loss": 1.1589,
+      "theoretical_loss": 3.507419589923717,
+      "tokens_seen": 1694302208
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.829080404429465e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.5073964126240105,
+      "tokens_seen": 1694433280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.828277965013641e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.5073732376190616,
+      "tokens_seen": 1694564352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.827475525597817e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.507350064908465,
+      "tokens_seen": 1694695424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.826673086181993e-05,
+      "loss": 1.3212,
+      "theoretical_loss": 3.507326894491817,
+      "tokens_seen": 1694826496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.82587064676617e-05,
+      "loss": 1.1807,
+      "theoretical_loss": 3.507303726368712,
+      "tokens_seen": 1694957568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.825068207350345e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.5072805605387467,
+      "tokens_seen": 1695088640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.824265767934521e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.507257397001516,
+      "tokens_seen": 1695219712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.823463328518697e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.5072342357566164,
+      "tokens_seen": 1695350784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.822660889102873e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.5072110768036433,
+      "tokens_seen": 1695481856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.821858449687049e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.507187920142193,
+      "tokens_seen": 1695612928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.821056010271224e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.5071647657718623,
+      "tokens_seen": 1695744000
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 570870,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2634421586990356,
+      "objective/train/theoretical_loss": 3.507141613692246,
+      "objective/train/tokens_used": 66334176,
+      "theoretical_loss": 3.507141613692246,
+      "tokens_seen": 1695875072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.820253570855401e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.507141613692246,
+      "tokens_seen": 1695875072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.819451131439576e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.507118463902942,
+      "tokens_seen": 1696006144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.818648692023753e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.507095316403545,
+      "tokens_seen": 1696137216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.817846252607928e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.5070721711936534,
+      "tokens_seen": 1696268288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.817043813192105e-05,
+      "loss": 1.3314,
+      "theoretical_loss": 3.5070490282728626,
+      "tokens_seen": 1696399360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.81624137377628e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.5070258876407703,
+      "tokens_seen": 1696530432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.815438934360455e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.507002749296973,
+      "tokens_seen": 1696661504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.814636494944632e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.5069796132410667,
+      "tokens_seen": 1696792576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.813834055528807e-05,
+      "loss": 1.162,
+      "theoretical_loss": 3.5069564794726498,
+      "tokens_seen": 1696923648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.813031616112984e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.5069333479913185,
+      "tokens_seen": 1697054720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.812229176697159e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.506910218796671,
+      "tokens_seen": 1697185792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.811426737281336e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.506887091888304,
+      "tokens_seen": 1697316864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.810624297865511e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.5068639672658155,
+      "tokens_seen": 1697447936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.809821858449687e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.506840844928803,
+      "tokens_seen": 1697579008
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.809019419033863e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.5068177248768633,
+      "tokens_seen": 1697710080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.808216979618039e-05,
+      "loss": 1.1717,
+      "theoretical_loss": 3.506794607109595,
+      "tokens_seen": 1697841152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.807414540202215e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.506771491626596,
+      "tokens_seen": 1697972224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.80661210078639e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.5067483784274645,
+      "tokens_seen": 1698103296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.805809661370567e-05,
+      "loss": 1.3237,
+      "theoretical_loss": 3.5067252675117975,
+      "tokens_seen": 1698234368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.805007221954742e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.506702158879194,
+      "tokens_seen": 1698365440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.804204782538918e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.5066790525292526,
+      "tokens_seen": 1698496512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.803402343123094e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.506655948461571,
+      "tokens_seen": 1698627584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.80259990370727e-05,
+      "loss": 1.2022,
+      "theoretical_loss": 3.506632846675748,
+      "tokens_seen": 1698758656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.801797464291446e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.5066097471713826,
+      "tokens_seen": 1698889728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.800995024875622e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.5065866499480727,
+      "tokens_seen": 1699020800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.800192585459798e-05,
+      "loss": 1.3136,
+      "theoretical_loss": 3.5065635550054175,
+      "tokens_seen": 1699151872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.799390146043974e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.5065404623430156,
+      "tokens_seen": 1699282944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.798587706628149e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.5065173719604665,
+      "tokens_seen": 1699414016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.797785267212326e-05,
+      "loss": 1.2799,
+      "theoretical_loss": 3.506494283857369,
+      "tokens_seen": 1699545088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.796982827796501e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.5064711980333225,
+      "tokens_seen": 1699676160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.796180388380678e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.5064481144879265,
+      "tokens_seen": 1699807232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.795377948964853e-05,
+      "loss": 1.176,
+      "theoretical_loss": 3.5064250332207796,
+      "tokens_seen": 1699938304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.79457550954903e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.506401954231482,
+      "tokens_seen": 1700069376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.793773070133205e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.5063788775196336,
+      "tokens_seen": 1700200448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.792970630717382e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.5063558030848325,
+      "tokens_seen": 1700331520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.792168191301557e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.5063327309266805,
+      "tokens_seen": 1700462592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.791365751885732e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.506309661044777,
+      "tokens_seen": 1700593664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.790563312469909e-05,
+      "loss": 1.1957,
+      "theoretical_loss": 3.506286593438721,
+      "tokens_seen": 1700724736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.789760873054084e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.5062635281081134,
+      "tokens_seen": 1700855808
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.788958433638261e-05,
+      "loss": 1.1735,
+      "theoretical_loss": 3.5062404650525547,
+      "tokens_seen": 1700986880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.788155994222436e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.5062174042716445,
+      "tokens_seen": 1701117952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.787353554806613e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.5061943457649836,
+      "tokens_seen": 1701249024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.786551115390788e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.5061712895321726,
+      "tokens_seen": 1701380096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.785748675974964e-05,
+      "loss": 1.1675,
+      "theoretical_loss": 3.506148235572812,
+      "tokens_seen": 1701511168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.78494623655914e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.5061251838865024,
+      "tokens_seen": 1701642240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.784143797143316e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.506102134472845,
+      "tokens_seen": 1701773312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.783341357727492e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.5060790873314405,
+      "tokens_seen": 1701904384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.782538918311668e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.506056042461889,
+      "tokens_seen": 1702035456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.781736478895844e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.506032999863794,
+      "tokens_seen": 1702166528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.78093403948002e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.506009959536754,
+      "tokens_seen": 1702297600
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/docs_used": 572947,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1634153127670288,
+      "objective/train/theoretical_loss": 3.5059869214803725,
+      "objective/train/tokens_used": 72887776,
+      "theoretical_loss": 3.5059869214803725,
+      "tokens_seen": 1702428672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.780131600064195e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.5059869214803725,
+      "tokens_seen": 1702428672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.779329160648371e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.50596388569425,
+      "tokens_seen": 1702559744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.778526721232547e-05,
+      "loss": 1.1784,
+      "theoretical_loss": 3.5059408521779876,
+      "tokens_seen": 1702690816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.777724281816723e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.505917820931187,
+      "tokens_seen": 1702821888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.776921842400899e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.505894791953451,
+      "tokens_seen": 1702952960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.776119402985075e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.5058717652443803,
+      "tokens_seen": 1703084032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.775316963569251e-05,
+      "loss": 1.1955,
+      "theoretical_loss": 3.5058487408035774,
+      "tokens_seen": 1703215104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.774514524153426e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.5058257186306445,
+      "tokens_seen": 1703346176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.773712084737603e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.505802698725183,
+      "tokens_seen": 1703477248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.772909645321778e-05,
+      "loss": 1.142,
+      "theoretical_loss": 3.505779681086796,
+      "tokens_seen": 1703608320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.772107205905955e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.5057566657150847,
+      "tokens_seen": 1703739392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.77130476649013e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.5057336526096528,
+      "tokens_seen": 1703870464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.770502327074307e-05,
+      "loss": 1.0959,
+      "theoretical_loss": 3.5057106417701025,
+      "tokens_seen": 1704001536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.769699887658482e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.5056876331960356,
+      "tokens_seen": 1704132608
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.768897448242657e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.5056646268870555,
+      "tokens_seen": 1704263680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.768095008826834e-05,
+      "loss": 1.1743,
+      "theoretical_loss": 3.5056416228427656,
+      "tokens_seen": 1704394752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.767292569411009e-05,
+      "loss": 1.1297,
+      "theoretical_loss": 3.5056186210627676,
+      "tokens_seen": 1704525824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.766490129995186e-05,
+      "loss": 1.1623,
+      "theoretical_loss": 3.5055956215466653,
+      "tokens_seen": 1704656896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.765687690579361e-05,
+      "loss": 1.2188,
+      "theoretical_loss": 3.5055726242940617,
+      "tokens_seen": 1704787968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.764885251163538e-05,
+      "loss": 1.1684,
+      "theoretical_loss": 3.50554962930456,
+      "tokens_seen": 1704919040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.764082811747713e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.5055266365777635,
+      "tokens_seen": 1705050112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.763280372331889e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.505503646113276,
+      "tokens_seen": 1705181184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.762477932916065e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.5054806579107,
+      "tokens_seen": 1705312256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.76167549350024e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.50545767196964,
+      "tokens_seen": 1705443328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.760873054084417e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.5054346882897,
+      "tokens_seen": 1705574400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.760070614668593e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.5054117068704835,
+      "tokens_seen": 1705705472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.759268175252769e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.505388727711594,
+      "tokens_seen": 1705836544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.758465735836945e-05,
+      "loss": 1.1053,
+      "theoretical_loss": 3.505365750812636,
+      "tokens_seen": 1705967616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.757663296421121e-05,
+      "loss": 1.1897,
+      "theoretical_loss": 3.5053427761732134,
+      "tokens_seen": 1706098688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.756860857005296e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.505319803792931,
+      "tokens_seen": 1706229760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.756058417589472e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.5052968336713923,
+      "tokens_seen": 1706360832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.755255978173648e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.505273865808202,
+      "tokens_seen": 1706491904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.754453538757824e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.505250900202965,
+      "tokens_seen": 1706622976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.753651099342e-05,
+      "loss": 1.1729,
+      "theoretical_loss": 3.5052279368552854,
+      "tokens_seen": 1706754048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.752848659926176e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.5052049757647685,
+      "tokens_seen": 1706885120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.752046220510352e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.505182016931019,
+      "tokens_seen": 1707016192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.751243781094528e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.5051590603536416,
+      "tokens_seen": 1707147264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.750441341678703e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.505136106032241,
+      "tokens_seen": 1707278336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.74963890226288e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.5051131539664233,
+      "tokens_seen": 1707409408
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.748836462847055e-05,
+      "loss": 1.2037,
+      "theoretical_loss": 3.5050902041557928,
+      "tokens_seen": 1707540480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 9.748034023431232e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.5050672565999554,
+      "tokens_seen": 1707671552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.747231584015407e-05,
+      "loss": 1.1735,
+      "theoretical_loss": 3.5050443112985166,
+      "tokens_seen": 1707802624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.746429144599584e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.5050213682510813,
+      "tokens_seen": 1707933696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.745626705183759e-05,
+      "loss": 1.2036,
+      "theoretical_loss": 3.504998427457255,
+      "tokens_seen": 1708064768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.744824265767934e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.504975488916645,
+      "tokens_seen": 1708195840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.744021826352111e-05,
+      "loss": 1.1234,
+      "theoretical_loss": 3.504952552628855,
+      "tokens_seen": 1708326912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.743219386936286e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.5049296185934926,
+      "tokens_seen": 1708457984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.742416947520463e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.504906686810163,
+      "tokens_seen": 1708589056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.741614508104638e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.504883757278473,
+      "tokens_seen": 1708720128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.740812068688815e-05,
+      "loss": 1.1534,
+      "theoretical_loss": 3.5048608299980275,
+      "tokens_seen": 1708851200
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 575138,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.279120683670044,
+      "objective/train/theoretical_loss": 3.5048379049684337,
+      "objective/train/tokens_used": 79441376,
+      "theoretical_loss": 3.5048379049684337,
+      "tokens_seen": 1708982272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.74000962927299e-05,
+      "loss": 1.1876,
+      "theoretical_loss": 3.5048379049684337,
+      "tokens_seen": 1708982272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.739207189857166e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.5048149821892984,
+      "tokens_seen": 1709113344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.738404750441342e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.504792061660228,
+      "tokens_seen": 1709244416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.737602311025518e-05,
+      "loss": 1.1764,
+      "theoretical_loss": 3.5047691433808286,
+      "tokens_seen": 1709375488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.736799871609694e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.5047462273507066,
+      "tokens_seen": 1709506560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.73599743219387e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.5047233135694698,
+      "tokens_seen": 1709637632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.735194992778046e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.5047004020367245,
+      "tokens_seen": 1709768704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.734392553362222e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.504677492752078,
+      "tokens_seen": 1709899776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.733590113946397e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.504654585715137,
+      "tokens_seen": 1710030848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.732787674530574e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.504631680925509,
+      "tokens_seen": 1710161920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.731985235114749e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.504608778382802,
+      "tokens_seen": 1710292992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.731182795698925e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.5045858780866217,
+      "tokens_seen": 1710424064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.730380356283101e-05,
+      "loss": 1.1295,
+      "theoretical_loss": 3.504562980036577,
+      "tokens_seen": 1710555136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.729577916867277e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.504540084232276,
+      "tokens_seen": 1710686208
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.728775477451453e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.5045171906733246,
+      "tokens_seen": 1710817280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.72797303803563e-05,
+      "loss": 1.1619,
+      "theoretical_loss": 3.5044942993593318,
+      "tokens_seen": 1710948352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.727170598619805e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.504471410289905,
+      "tokens_seen": 1711079424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.72636815920398e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.504448523464653,
+      "tokens_seen": 1711210496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.725565719788157e-05,
+      "loss": 1.133,
+      "theoretical_loss": 3.5044256388831827,
+      "tokens_seen": 1711341568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.724763280372332e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.5044027565451032,
+      "tokens_seen": 1711472640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.723960840956509e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.5043798764500225,
+      "tokens_seen": 1711603712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.723158401540684e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.504356998597549,
+      "tokens_seen": 1711734784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.722355962124861e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.5043341229872915,
+      "tokens_seen": 1711865856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.721553522709036e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.504311249618858,
+      "tokens_seen": 1711996928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.720751083293211e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.5042883784918577,
+      "tokens_seen": 1712128000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.719948643877388e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.504265509605899,
+      "tokens_seen": 1712259072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.719146204461563e-05,
+      "loss": 1.118,
+      "theoretical_loss": 3.504242642960591,
+      "tokens_seen": 1712390144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.71834376504574e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.5042197785555427,
+      "tokens_seen": 1712521216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.717541325629915e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.504196916390363,
+      "tokens_seen": 1712652288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.716738886214092e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.5041740564646613,
+      "tokens_seen": 1712783360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.715936446798267e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.504151198778047,
+      "tokens_seen": 1712914432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.715134007382443e-05,
+      "loss": 1.132,
+      "theoretical_loss": 3.5041283433301293,
+      "tokens_seen": 1713045504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.714331567966619e-05,
+      "loss": 1.1733,
+      "theoretical_loss": 3.5041054901205175,
+      "tokens_seen": 1713176576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.713529128550795e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.5040826391488213,
+      "tokens_seen": 1713307648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.712726689134971e-05,
+      "loss": 1.1791,
+      "theoretical_loss": 3.50405979041465,
+      "tokens_seen": 1713438720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.711924249719147e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.5040369439176144,
+      "tokens_seen": 1713569792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.711121810303323e-05,
+      "loss": 1.1962,
+      "theoretical_loss": 3.504014099657323,
+      "tokens_seen": 1713700864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.710319370887499e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.5039912576333876,
+      "tokens_seen": 1713831936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.709516931471674e-05,
+      "loss": 1.1726,
+      "theoretical_loss": 3.5039684178454165,
+      "tokens_seen": 1713963008
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.70871449205585e-05,
+      "loss": 1.1722,
+      "theoretical_loss": 3.50394558029302,
+      "tokens_seen": 1714094080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.707912052640026e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.5039227449758092,
+      "tokens_seen": 1714225152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.707109613224202e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.5038999118933942,
+      "tokens_seen": 1714356224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.706307173808378e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.5038770810453856,
+      "tokens_seen": 1714487296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.705504734392554e-05,
+      "loss": 1.2784,
+      "theoretical_loss": 3.5038542524313936,
+      "tokens_seen": 1714618368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.70470229497673e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.503831426051029,
+      "tokens_seen": 1714749440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.703899855560905e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.5038086019039025,
+      "tokens_seen": 1714880512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.703097416145082e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.5037857799896246,
+      "tokens_seen": 1715011584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.702294976729257e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.503762960307807,
+      "tokens_seen": 1715142656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.701492537313434e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.50374014285806,
+      "tokens_seen": 1715273728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.700690097897609e-05,
+      "loss": 1.3126,
+      "theoretical_loss": 3.5037173276399955,
+      "tokens_seen": 1715404800
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 577290,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.134467363357544,
+      "objective/train/theoretical_loss": 3.5036945146532243,
+      "objective/train/tokens_used": 85994976,
+      "theoretical_loss": 3.5036945146532243,
+      "tokens_seen": 1715535872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.699887658481786e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.5036945146532243,
+      "tokens_seen": 1715535872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.699085219065961e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.503671703897357,
+      "tokens_seen": 1715666944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.698282779650138e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.503648895372007,
+      "tokens_seen": 1715798016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.697480340234313e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.503626089076784,
+      "tokens_seen": 1715929088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.696677900818488e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.5036032850113,
+      "tokens_seen": 1716060160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.695875461402665e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.503580483175167,
+      "tokens_seen": 1716191232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.69507302198684e-05,
+      "loss": 1.1783,
+      "theoretical_loss": 3.5035576835679967,
+      "tokens_seen": 1716322304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.694270582571017e-05,
+      "loss": 1.1882,
+      "theoretical_loss": 3.503534886189401,
+      "tokens_seen": 1716453376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.693468143155192e-05,
+      "loss": 1.1739,
+      "theoretical_loss": 3.503512091038992,
+      "tokens_seen": 1716584448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.692665703739369e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.503489298116382,
+      "tokens_seen": 1716715520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.691863264323544e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.503466507421183,
+      "tokens_seen": 1716846592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.69106082490772e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.5034437189530068,
+      "tokens_seen": 1716977664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.690258385491896e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.5034209327114665,
+      "tokens_seen": 1717108736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.689455946076072e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.5033981486961747,
+      "tokens_seen": 1717239808
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.688653506660248e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.5033753669067433,
+      "tokens_seen": 1717370880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.687851067244424e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.5033525873427855,
+      "tokens_seen": 1717501952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.6870486278286e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.5033298100039145,
+      "tokens_seen": 1717633024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.686246188412776e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.503307034889742,
+      "tokens_seen": 1717764096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.685443748996951e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.503284261999882,
+      "tokens_seen": 1717895168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.684641309581128e-05,
+      "loss": 1.1898,
+      "theoretical_loss": 3.503261491333947,
+      "tokens_seen": 1718026240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.683838870165303e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.5032387228915507,
+      "tokens_seen": 1718157312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.68303643074948e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.503215956672306,
+      "tokens_seen": 1718288384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.682233991333655e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.5031931926758255,
+      "tokens_seen": 1718419456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.681431551917831e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.503170430901724,
+      "tokens_seen": 1718550528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.680629112502007e-05,
+      "loss": 1.1697,
+      "theoretical_loss": 3.503147671349615,
+      "tokens_seen": 1718681600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.679826673086182e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.503124914019111,
+      "tokens_seen": 1718812672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.679024233670359e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.503102158909827,
+      "tokens_seen": 1718943744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.678221794254534e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.503079406021376,
+      "tokens_seen": 1719074816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.677419354838711e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.5030566553533724,
+      "tokens_seen": 1719205888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.676616915422886e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.50303390690543,
+      "tokens_seen": 1719336960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.675814476007063e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.503011160677164,
+      "tokens_seen": 1719468032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.675012036591238e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.5029884166681864,
+      "tokens_seen": 1719599104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.674209597175413e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.502965674878113,
+      "tokens_seen": 1719730176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.67340715775959e-05,
+      "loss": 1.3138,
+      "theoretical_loss": 3.5029429353065584,
+      "tokens_seen": 1719861248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.672604718343765e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.502920197953137,
+      "tokens_seen": 1719992320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.671802278927942e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.502897462817463,
+      "tokens_seen": 1720123392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.670999839512117e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.502874729899151,
+      "tokens_seen": 1720254464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.670197400096294e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.5028519991978166,
+      "tokens_seen": 1720385536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.669394960680469e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.5028292707130744,
+      "tokens_seen": 1720516608
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.668592521264645e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.502806544444539,
+      "tokens_seen": 1720647680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.667790081848821e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.5027838203918256,
+      "tokens_seen": 1720778752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.666987642432997e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.50276109855455,
+      "tokens_seen": 1720909824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.666185203017173e-05,
+      "loss": 1.1505,
+      "theoretical_loss": 3.5027383789323263,
+      "tokens_seen": 1721040896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.665382763601349e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.502715661524771,
+      "tokens_seen": 1721171968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.664580324185525e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.5026929463315,
+      "tokens_seen": 1721303040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.6637778847697e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.5026702333521276,
+      "tokens_seen": 1721434112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.662975445353877e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.50264752258627,
+      "tokens_seen": 1721565184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.662173005938053e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.5026248140335428,
+      "tokens_seen": 1721696256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.661370566522228e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.5026021076935625,
+      "tokens_seen": 1721827328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.660568127106405e-05,
+      "loss": 1.116,
+      "theoretical_loss": 3.502579403565944,
+      "tokens_seen": 1721958400
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/docs_used": 579595,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2168859243392944,
+      "objective/train/theoretical_loss": 3.502556701650305,
+      "objective/train/tokens_used": 92548576,
+      "theoretical_loss": 3.502556701650305,
+      "tokens_seen": 1722089472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.65976568769058e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.502556701650305,
+      "tokens_seen": 1722089472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.658963248274757e-05,
+      "loss": 1.1868,
+      "theoretical_loss": 3.50253400194626,
+      "tokens_seen": 1722220544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.658160808858932e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.502511304453426,
+      "tokens_seen": 1722351616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.657358369443108e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.5024886091714196,
+      "tokens_seen": 1722482688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.656555930027284e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.5024659160998564,
+      "tokens_seen": 1722613760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.655753490611459e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.5024432252383537,
+      "tokens_seen": 1722744832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.654951051195636e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.502420536586528,
+      "tokens_seen": 1722875904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.654148611779811e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.5023978501439963,
+      "tokens_seen": 1723006976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.653346172363988e-05,
+      "loss": 1.1997,
+      "theoretical_loss": 3.502375165910374,
+      "tokens_seen": 1723138048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.652543732948163e-05,
+      "loss": 1.1633,
+      "theoretical_loss": 3.5023524838852804,
+      "tokens_seen": 1723269120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.65174129353234e-05,
+      "loss": 1.1632,
+      "theoretical_loss": 3.5023298040683306,
+      "tokens_seen": 1723400192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.650938854116515e-05,
+      "loss": 1.1639,
+      "theoretical_loss": 3.5023071264591423,
+      "tokens_seen": 1723531264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.65013641470069e-05,
+      "loss": 1.1711,
+      "theoretical_loss": 3.5022844510573328,
+      "tokens_seen": 1723662336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.649333975284867e-05,
+      "loss": 1.1625,
+      "theoretical_loss": 3.502261777862519,
+      "tokens_seen": 1723793408
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.648531535869042e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.5022391068743195,
+      "tokens_seen": 1723924480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.647729096453219e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.5022164380923506,
+      "tokens_seen": 1724055552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 9.646926657037394e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.5021937715162306,
+      "tokens_seen": 1724186624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.646124217621571e-05,
+      "loss": 1.1726,
+      "theoretical_loss": 3.5021711071455766,
+      "tokens_seen": 1724317696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.645321778205746e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.502148444980006,
+      "tokens_seen": 1724448768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.644519338789922e-05,
+      "loss": 1.1684,
+      "theoretical_loss": 3.502125785019138,
+      "tokens_seen": 1724579840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.643716899374098e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.5021031272625898,
+      "tokens_seen": 1724710912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.642914459958274e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.50208047170998,
+      "tokens_seen": 1724841984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.64211202054245e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.502057818360926,
+      "tokens_seen": 1724973056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.641309581126626e-05,
+      "loss": 1.0971,
+      "theoretical_loss": 3.502035167215046,
+      "tokens_seen": 1725104128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.640507141710802e-05,
+      "loss": 1.3375,
+      "theoretical_loss": 3.5020125182719593,
+      "tokens_seen": 1725235200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.639704702294978e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.5019898715312836,
+      "tokens_seen": 1725366272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.638902262879153e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.5019672269926376,
+      "tokens_seen": 1725497344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.63809982346333e-05,
+      "loss": 1.2111,
+      "theoretical_loss": 3.5019445846556403,
+      "tokens_seen": 1725628416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.637297384047505e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.5019219445199097,
+      "tokens_seen": 1725759488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.636494944631682e-05,
+      "loss": 1.1815,
+      "theoretical_loss": 3.5018993065850657,
+      "tokens_seen": 1725890560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.635692505215857e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.501876670850726,
+      "tokens_seen": 1726021632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.634890065800034e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.50185403731651,
+      "tokens_seen": 1726152704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.634087626384209e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.501831405982038,
+      "tokens_seen": 1726283776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.633285186968385e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.5018087768469277,
+      "tokens_seen": 1726414848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.632482747552561e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.5017861499107985,
+      "tokens_seen": 1726545920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.631680308136736e-05,
+      "loss": 1.1871,
+      "theoretical_loss": 3.501763525173271,
+      "tokens_seen": 1726676992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.630877868720913e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.501740902633964,
+      "tokens_seen": 1726808064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.630075429305088e-05,
+      "loss": 1.1394,
+      "theoretical_loss": 3.5017182822924964,
+      "tokens_seen": 1726939136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.629272989889265e-05,
+      "loss": 1.139,
+      "theoretical_loss": 3.501695664148489,
+      "tokens_seen": 1727070208
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.62847055047344e-05,
+      "loss": 1.1786,
+      "theoretical_loss": 3.5016730482015612,
+      "tokens_seen": 1727201280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.627668111057617e-05,
+      "loss": 1.1686,
+      "theoretical_loss": 3.5016504344513324,
+      "tokens_seen": 1727332352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.626865671641792e-05,
+      "loss": 1.145,
+      "theoretical_loss": 3.501627822897423,
+      "tokens_seen": 1727463424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.626063232225967e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.5016052135394533,
+      "tokens_seen": 1727594496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.625260792810144e-05,
+      "loss": 1.1551,
+      "theoretical_loss": 3.501582606377043,
+      "tokens_seen": 1727725568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.62445835339432e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.5015600014098127,
+      "tokens_seen": 1727856640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.623655913978496e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.5015373986373826,
+      "tokens_seen": 1727987712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.622853474562671e-05,
+      "loss": 1.1267,
+      "theoretical_loss": 3.5015147980593726,
+      "tokens_seen": 1728118784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.622051035146848e-05,
+      "loss": 1.1244,
+      "theoretical_loss": 3.5014921996754045,
+      "tokens_seen": 1728249856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.621248595731023e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.5014696034850976,
+      "tokens_seen": 1728380928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.620446156315199e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.5014470094880736,
+      "tokens_seen": 1728512000
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 581638,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1808078289031982,
+      "objective/train/theoretical_loss": 3.501424417683953,
+      "objective/train/tokens_used": 99102176,
+      "theoretical_loss": 3.501424417683953,
+      "tokens_seen": 1728643072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.619643716899375e-05,
+      "loss": 1.1814,
+      "theoretical_loss": 3.501424417683953,
+      "tokens_seen": 1728643072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.61884127748355e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.501401828072356,
+      "tokens_seen": 1728774144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.618038838067727e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.5013792406529047,
+      "tokens_seen": 1728905216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.617236398651903e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.5013566554252202,
+      "tokens_seen": 1729036288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.616433959236079e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.501334072388923,
+      "tokens_seen": 1729167360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.615631519820255e-05,
+      "loss": 1.1111,
+      "theoretical_loss": 3.5013114915436345,
+      "tokens_seen": 1729298432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.61482908040443e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.5012889128889757,
+      "tokens_seen": 1729429504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.614026640988607e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.5012663364245693,
+      "tokens_seen": 1729560576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.613224201572782e-05,
+      "loss": 1.1077,
+      "theoretical_loss": 3.501243762150036,
+      "tokens_seen": 1729691648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.612421762156959e-05,
+      "loss": 1.1815,
+      "theoretical_loss": 3.5012211900649977,
+      "tokens_seen": 1729822720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.611619322741134e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.501198620169076,
+      "tokens_seen": 1729953792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.610816883325309e-05,
+      "loss": 1.1175,
+      "theoretical_loss": 3.501176052461893,
+      "tokens_seen": 1730084864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.610014443909486e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.5011534869430703,
+      "tokens_seen": 1730215936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.609212004493661e-05,
+      "loss": 1.089,
+      "theoretical_loss": 3.5011309236122305,
+      "tokens_seen": 1730347008
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.608409565077838e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.5011083624689956,
+      "tokens_seen": 1730478080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.607607125662013e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.501085803512987,
+      "tokens_seen": 1730609152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.60680468624619e-05,
+      "loss": 1.2766,
+      "theoretical_loss": 3.5010632467438283,
+      "tokens_seen": 1730740224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.606002246830365e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.5010406921611406,
+      "tokens_seen": 1730871296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.60519980741454e-05,
+      "loss": 1.2338,
+      "theoretical_loss": 3.5010181397645472,
+      "tokens_seen": 1731002368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.604397367998717e-05,
+      "loss": 1.143,
+      "theoretical_loss": 3.500995589553671,
+      "tokens_seen": 1731133440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.603594928582892e-05,
+      "loss": 1.1312,
+      "theoretical_loss": 3.500973041528134,
+      "tokens_seen": 1731264512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.602792489167069e-05,
+      "loss": 1.1255,
+      "theoretical_loss": 3.5009504956875594,
+      "tokens_seen": 1731395584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.601990049751244e-05,
+      "loss": 1.1364,
+      "theoretical_loss": 3.50092795203157,
+      "tokens_seen": 1731526656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.60118761033542e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.5009054105597883,
+      "tokens_seen": 1731657728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.600385170919596e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.5008828712718385,
+      "tokens_seen": 1731788800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.599582731503772e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.500860334167343,
+      "tokens_seen": 1731919872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.598780292087948e-05,
+      "loss": 1.1089,
+      "theoretical_loss": 3.5008377992459248,
+      "tokens_seen": 1732050944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.597977852672124e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.5008152665072076,
+      "tokens_seen": 1732182016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.5971754132563e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.500792735950815,
+      "tokens_seen": 1732313088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.596372973840476e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.5007702075763705,
+      "tokens_seen": 1732444160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.595570534424651e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.5007476813834977,
+      "tokens_seen": 1732575232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.594768095008828e-05,
+      "loss": 1.1564,
+      "theoretical_loss": 3.5007251573718197,
+      "tokens_seen": 1732706304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.593965655593003e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.5007026355409616,
+      "tokens_seen": 1732837376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.59316321617718e-05,
+      "loss": 1.1146,
+      "theoretical_loss": 3.500680115890546,
+      "tokens_seen": 1732968448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.592360776761355e-05,
+      "loss": 1.2063,
+      "theoretical_loss": 3.500657598420198,
+      "tokens_seen": 1733099520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.59155833734553e-05,
+      "loss": 1.1458,
+      "theoretical_loss": 3.5006350831295414,
+      "tokens_seen": 1733230592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.590755897929707e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.5006125700181996,
+      "tokens_seen": 1733361664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.589953458513882e-05,
+      "loss": 1.1122,
+      "theoretical_loss": 3.5005900590857975,
+      "tokens_seen": 1733492736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.589151019098059e-05,
+      "loss": 1.1548,
+      "theoretical_loss": 3.5005675503319598,
+      "tokens_seen": 1733623808
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.588348579682234e-05,
+      "loss": 1.1743,
+      "theoretical_loss": 3.5005450437563104,
+      "tokens_seen": 1733754880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.587546140266411e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.5005225393584745,
+      "tokens_seen": 1733885952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.586743700850586e-05,
+      "loss": 1.0943,
+      "theoretical_loss": 3.5005000371380763,
+      "tokens_seen": 1734017024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.585941261434761e-05,
+      "loss": 1.101,
+      "theoretical_loss": 3.5004775370947403,
+      "tokens_seen": 1734148096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.585138822018938e-05,
+      "loss": 1.1717,
+      "theoretical_loss": 3.500455039228092,
+      "tokens_seen": 1734279168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.584336382603113e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.500432543537756,
+      "tokens_seen": 1734410240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.58353394318729e-05,
+      "loss": 1.1686,
+      "theoretical_loss": 3.5004100500233575,
+      "tokens_seen": 1734541312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.582731503771465e-05,
+      "loss": 1.209,
+      "theoretical_loss": 3.500387558684521,
+      "tokens_seen": 1734672384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.581929064355641e-05,
+      "loss": 1.165,
+      "theoretical_loss": 3.500365069520872,
+      "tokens_seen": 1734803456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.581126624939817e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.5003425825320367,
+      "tokens_seen": 1734934528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.580324185523993e-05,
+      "loss": 1.1599,
+      "theoretical_loss": 3.5003200977176396,
+      "tokens_seen": 1735065600
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/docs_used": 583892,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2191352844238281,
+      "objective/train/theoretical_loss": 3.500297615077306,
+      "objective/train/tokens_used": 105655776,
+      "theoretical_loss": 3.500297615077306,
+      "tokens_seen": 1735196672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.57952174610817e-05,
+      "loss": 1.2758,
+      "theoretical_loss": 3.500297615077306,
+      "tokens_seen": 1735196672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.578719306692345e-05,
+      "loss": 1.1522,
+      "theoretical_loss": 3.5002751346106624,
+      "tokens_seen": 1735327744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.57791686727652e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.5002526563173344,
+      "tokens_seen": 1735458816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.577114427860697e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.5002301801969464,
+      "tokens_seen": 1735589888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.576311988444872e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.500207706249126,
+      "tokens_seen": 1735720960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.575509549029049e-05,
+      "loss": 1.2348,
+      "theoretical_loss": 3.500185234473498,
+      "tokens_seen": 1735852032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.574707109613224e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.500162764869689,
+      "tokens_seen": 1735983104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.5739046701974e-05,
+      "loss": 1.1538,
+      "theoretical_loss": 3.5001402974373246,
+      "tokens_seen": 1736114176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.573102230781576e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.500117832176032,
+      "tokens_seen": 1736245248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.572299791365751e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.500095369085437,
+      "tokens_seen": 1736376320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.571497351949928e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.5000729081651656,
+      "tokens_seen": 1736507392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.570694912534103e-05,
+      "loss": 1.1042,
+      "theoretical_loss": 3.5000504494148448,
+      "tokens_seen": 1736638464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.56989247311828e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.5000279928341005,
+      "tokens_seen": 1736769536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.569090033702455e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.5000055384225606,
+      "tokens_seen": 1736900608
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.568287594286632e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.4999830861798515,
+      "tokens_seen": 1737031680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.567485154870807e-05,
+      "loss": 1.1185,
+      "theoretical_loss": 3.4999606361056,
+      "tokens_seen": 1737162752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.566682715454983e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.4999381881994323,
+      "tokens_seen": 1737293824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.565880276039159e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.4999157424609764,
+      "tokens_seen": 1737424896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.565077836623335e-05,
+      "loss": 1.1897,
+      "theoretical_loss": 3.4998932988898597,
+      "tokens_seen": 1737555968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.564275397207511e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.4998708574857083,
+      "tokens_seen": 1737687040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.563472957791687e-05,
+      "loss": 1.1939,
+      "theoretical_loss": 3.4998484182481504,
+      "tokens_seen": 1737818112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.562670518375863e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.499825981176813,
+      "tokens_seen": 1737949184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.561868078960038e-05,
+      "loss": 1.1427,
+      "theoretical_loss": 3.499803546271324,
+      "tokens_seen": 1738080256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.561065639544214e-05,
+      "loss": 1.1097,
+      "theoretical_loss": 3.4997811135313115,
+      "tokens_seen": 1738211328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.56026320012839e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.4997586829564025,
+      "tokens_seen": 1738342400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.559460760712566e-05,
+      "loss": 1.2911,
+      "theoretical_loss": 3.4997362545462245,
+      "tokens_seen": 1738473472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.558658321296742e-05,
+      "loss": 1.1853,
+      "theoretical_loss": 3.499713828300406,
+      "tokens_seen": 1738604544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.557855881880918e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.4996914042185745,
+      "tokens_seen": 1738735616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.557053442465094e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.4996689823003586,
+      "tokens_seen": 1738866688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.55625100304927e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.4996465625453865,
+      "tokens_seen": 1738997760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.555448563633445e-05,
+      "loss": 1.1682,
+      "theoretical_loss": 3.4996241449532857,
+      "tokens_seen": 1739128832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.554646124217622e-05,
+      "loss": 1.1665,
+      "theoretical_loss": 3.4996017295236856,
+      "tokens_seen": 1739259904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.553843684801797e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.499579316256214,
+      "tokens_seen": 1739390976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.553041245385974e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.4995569051504996,
+      "tokens_seen": 1739522048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.552238805970149e-05,
+      "loss": 1.1585,
+      "theoretical_loss": 3.499534496206171,
+      "tokens_seen": 1739653120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.551436366554326e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.499512089422857,
+      "tokens_seen": 1739784192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.550633927138501e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.499489684800186,
+      "tokens_seen": 1739915264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.549831487722676e-05,
+      "loss": 1.1499,
+      "theoretical_loss": 3.499467282337788,
+      "tokens_seen": 1740046336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.549029048306853e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.4994448820352906,
+      "tokens_seen": 1740177408
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.548226608891028e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.499422483892324,
+      "tokens_seen": 1740308480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.547424169475205e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.499400087908517,
+      "tokens_seen": 1740439552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.54662173005938e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.4993776940834986,
+      "tokens_seen": 1740570624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.545819290643557e-05,
+      "loss": 1.1321,
+      "theoretical_loss": 3.4993553024168986,
+      "tokens_seen": 1740701696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.545016851227732e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.499332912908346,
+      "tokens_seen": 1740832768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.544214411811908e-05,
+      "loss": 1.1393,
+      "theoretical_loss": 3.499310525557471,
+      "tokens_seen": 1740963840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.543411972396084e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.4992881403639027,
+      "tokens_seen": 1741094912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.54260953298026e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.499265757327271,
+      "tokens_seen": 1741225984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.541807093564436e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.4992433764472057,
+      "tokens_seen": 1741357056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.541004654148612e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.499220997723337,
+      "tokens_seen": 1741488128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.540202214732788e-05,
+      "loss": 1.1695,
+      "theoretical_loss": 3.499198621155294,
+      "tokens_seen": 1741619200
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 586110,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.144083023071289,
+      "objective/train/theoretical_loss": 3.4991762467427083,
+      "objective/train/tokens_used": 112209376,
+      "theoretical_loss": 3.4991762467427083,
+      "tokens_seen": 1741750272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.539399775316964e-05,
+      "loss": 1.1325,
+      "theoretical_loss": 3.4991762467427083,
+      "tokens_seen": 1741750272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.53859733590114e-05,
+      "loss": 1.136,
+      "theoretical_loss": 3.4991538744852084,
+      "tokens_seen": 1741881344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.537794896485315e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.499131504382426,
+      "tokens_seen": 1742012416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.536992457069491e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.499109136433991,
+      "tokens_seen": 1742143488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.536190017653667e-05,
+      "loss": 1.1452,
+      "theoretical_loss": 3.4990867706395337,
+      "tokens_seen": 1742274560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.535387578237843e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.4990644069986843,
+      "tokens_seen": 1742405632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.53458513882202e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.4990420455110742,
+      "tokens_seen": 1742536704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.533782699406195e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.4990196861763336,
+      "tokens_seen": 1742667776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.532980259990371e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4989973289940934,
+      "tokens_seen": 1742798848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.532177820574547e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.498974973963985,
+      "tokens_seen": 1742929920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.531375381158722e-05,
+      "loss": 1.1618,
+      "theoretical_loss": 3.498952621085639,
+      "tokens_seen": 1743060992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.530572941742899e-05,
+      "loss": 1.1793,
+      "theoretical_loss": 3.4989302703586866,
+      "tokens_seen": 1743192064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.529770502327074e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.498907921782759,
+      "tokens_seen": 1743323136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.528968062911251e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4988855753574875,
+      "tokens_seen": 1743454208
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.528165623495426e-05,
+      "loss": 1.0992,
+      "theoretical_loss": 3.4988632310825034,
+      "tokens_seen": 1743585280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.527363184079603e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.498840888957438,
+      "tokens_seen": 1743716352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.526560744663778e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.498818548981923,
+      "tokens_seen": 1743847424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.525758305247953e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.4987962111555904,
+      "tokens_seen": 1743978496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.52495586583213e-05,
+      "loss": 1.1586,
+      "theoretical_loss": 3.4987738754780713,
+      "tokens_seen": 1744109568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.524153426416305e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.498751541948998,
+      "tokens_seen": 1744240640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.523350987000482e-05,
+      "loss": 1.141,
+      "theoretical_loss": 3.498729210568002,
+      "tokens_seen": 1744371712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.522548547584657e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.498706881334716,
+      "tokens_seen": 1744502784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.521746108168834e-05,
+      "loss": 1.1863,
+      "theoretical_loss": 3.498684554248772,
+      "tokens_seen": 1744633856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.520943668753009e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.4986622293098013,
+      "tokens_seen": 1744764928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.520141229337185e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.498639906517437,
+      "tokens_seen": 1744896000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.519338789921361e-05,
+      "loss": 1.1394,
+      "theoretical_loss": 3.498617585871311,
+      "tokens_seen": 1745027072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.518536350505537e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.4985952673710567,
+      "tokens_seen": 1745158144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.517733911089713e-05,
+      "loss": 1.1497,
+      "theoretical_loss": 3.498572951016305,
+      "tokens_seen": 1745289216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.516931471673889e-05,
+      "loss": 1.1756,
+      "theoretical_loss": 3.4985506368066903,
+      "tokens_seen": 1745420288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.516129032258065e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.4985283247418444,
+      "tokens_seen": 1745551360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.51532659284224e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.4985060148214,
+      "tokens_seen": 1745682432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.514524153426416e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.4984837070449903,
+      "tokens_seen": 1745813504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.513721714010593e-05,
+      "loss": 1.1623,
+      "theoretical_loss": 3.498461401412248,
+      "tokens_seen": 1745944576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.512919274594768e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.498439097922807,
+      "tokens_seen": 1746075648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.512116835178944e-05,
+      "loss": 1.1302,
+      "theoretical_loss": 3.4984167965763002,
+      "tokens_seen": 1746206720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.51131439576312e-05,
+      "loss": 1.1464,
+      "theoretical_loss": 3.4983944973723604,
+      "tokens_seen": 1746337792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.510511956347296e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4983722003106212,
+      "tokens_seen": 1746468864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.509709516931472e-05,
+      "loss": 1.1607,
+      "theoretical_loss": 3.498349905390716,
+      "tokens_seen": 1746599936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.508907077515648e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4983276126122784,
+      "tokens_seen": 1746731008
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.508104638099824e-05,
+      "loss": 1.0872,
+      "theoretical_loss": 3.498305321974942,
+      "tokens_seen": 1746862080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.507302198683999e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.498283033478341,
+      "tokens_seen": 1746993152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.506499759268176e-05,
+      "loss": 1.1508,
+      "theoretical_loss": 3.4982607471221083,
+      "tokens_seen": 1747124224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.505697319852351e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.4982384629058787,
+      "tokens_seen": 1747255296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.504894880436528e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.498216180829286,
+      "tokens_seen": 1747386368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.504092441020703e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.498193900891964,
+      "tokens_seen": 1747517440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.50329000160488e-05,
+      "loss": 1.1491,
+      "theoretical_loss": 3.4981716230935467,
+      "tokens_seen": 1747648512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.502487562189055e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.498149347433669,
+      "tokens_seen": 1747779584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.50168512277323e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.4981270739119648,
+      "tokens_seen": 1747910656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.500882683357407e-05,
+      "loss": 1.1653,
+      "theoretical_loss": 3.4981048025280685,
+      "tokens_seen": 1748041728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.500080243941582e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.498082533281615,
+      "tokens_seen": 1748172800
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 588356,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.103625774383545,
+      "objective/train/theoretical_loss": 3.4980602661722386,
+      "objective/train/tokens_used": 118762976,
+      "theoretical_loss": 3.4980602661722386,
+      "tokens_seen": 1748303872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.499277804525759e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.4980602661722386,
+      "tokens_seen": 1748303872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.498475365109934e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.498038001199574,
+      "tokens_seen": 1748434944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.497672925694111e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4980157383632564,
+      "tokens_seen": 1748566016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.496870486278286e-05,
+      "loss": 1.1717,
+      "theoretical_loss": 3.497993477662921,
+      "tokens_seen": 1748697088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.496068046862462e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.4979712190982015,
+      "tokens_seen": 1748828160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.495265607446638e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.497948962668734,
+      "tokens_seen": 1748959232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.494463168030814e-05,
+      "loss": 1.1199,
+      "theoretical_loss": 3.4979267083741536,
+      "tokens_seen": 1749090304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.49366072861499e-05,
+      "loss": 1.1624,
+      "theoretical_loss": 3.497904456214095,
+      "tokens_seen": 1749221376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.492858289199166e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.4978822061881942,
+      "tokens_seen": 1749352448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.492055849783342e-05,
+      "loss": 1.168,
+      "theoretical_loss": 3.497859958296086,
+      "tokens_seen": 1749483520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.491253410367518e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4978377125374065,
+      "tokens_seen": 1749614592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.490450970951693e-05,
+      "loss": 1.1432,
+      "theoretical_loss": 3.4978154689117913,
+      "tokens_seen": 1749745664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.48964853153587e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.4977932274188754,
+      "tokens_seen": 1749876736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.488846092120045e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.4977709880582957,
+      "tokens_seen": 1750007808
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.488043652704221e-05,
+      "loss": 1.1862,
+      "theoretical_loss": 3.497748750829687,
+      "tokens_seen": 1750138880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.487241213288397e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.497726515732686,
+      "tokens_seen": 1750269952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.486438773872573e-05,
+      "loss": 1.1557,
+      "theoretical_loss": 3.4977042827669282,
+      "tokens_seen": 1750401024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.485636334456749e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.4976820519320504,
+      "tokens_seen": 1750532096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.484833895040924e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.4976598232276883,
+      "tokens_seen": 1750663168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.484031455625101e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.4976375966534787,
+      "tokens_seen": 1750794240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.483229016209276e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.497615372209057,
+      "tokens_seen": 1750925312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.482426576793453e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.4975931498940613,
+      "tokens_seen": 1751056384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.481624137377628e-05,
+      "loss": 1.1571,
+      "theoretical_loss": 3.497570929708127,
+      "tokens_seen": 1751187456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.480821697961805e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.497548711650891,
+      "tokens_seen": 1751318528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.48001925854598e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.4975264957219907,
+      "tokens_seen": 1751449600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.479216819130155e-05,
+      "loss": 1.1841,
+      "theoretical_loss": 3.4975042819210627,
+      "tokens_seen": 1751580672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.478414379714332e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.497482070247743,
+      "tokens_seen": 1751711744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.477611940298507e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4974598607016696,
+      "tokens_seen": 1751842816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.476809500882684e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.4974376532824794,
+      "tokens_seen": 1751973888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.476007061466859e-05,
+      "loss": 1.2653,
+      "theoretical_loss": 3.49741544798981,
+      "tokens_seen": 1752104960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.475204622051036e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.4973932448232983,
+      "tokens_seen": 1752236032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.474402182635211e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4973710437825813,
+      "tokens_seen": 1752367104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.473599743219388e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.497348844867297,
+      "tokens_seen": 1752498176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.472797303803563e-05,
+      "loss": 1.2102,
+      "theoretical_loss": 3.4973266480770837,
+      "tokens_seen": 1752629248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.471994864387739e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.4973044534115774,
+      "tokens_seen": 1752760320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.471192424971915e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.497282260870417,
+      "tokens_seen": 1752891392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.47038998555609e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.4972600704532404,
+      "tokens_seen": 1753022464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.469587546140267e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.497237882159685,
+      "tokens_seen": 1753153536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.468785106724443e-05,
+      "loss": 1.1729,
+      "theoretical_loss": 3.4972156959893894,
+      "tokens_seen": 1753284608
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.467982667308619e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.497193511941991,
+      "tokens_seen": 1753415680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.467180227892795e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.497171330017128,
+      "tokens_seen": 1753546752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.46637778847697e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.49714915021444,
+      "tokens_seen": 1753677824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.465575349061147e-05,
+      "loss": 1.1663,
+      "theoretical_loss": 3.4971269725335636,
+      "tokens_seen": 1753808896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.464772909645322e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.4971047969741385,
+      "tokens_seen": 1753939968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.463970470229499e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4970826235358032,
+      "tokens_seen": 1754071040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.463168030813674e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4970604522181956,
+      "tokens_seen": 1754202112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.46236559139785e-05,
+      "loss": 1.1813,
+      "theoretical_loss": 3.497038283020955,
+      "tokens_seen": 1754333184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.461563151982026e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.49701611594372,
+      "tokens_seen": 1754464256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.460760712566201e-05,
+      "loss": 1.1906,
+      "theoretical_loss": 3.496993950986129,
+      "tokens_seen": 1754595328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.459958273150378e-05,
+      "loss": 1.1919,
+      "theoretical_loss": 3.496971788147822,
+      "tokens_seen": 1754726400
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/docs_used": 590929,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1672173738479614,
+      "objective/train/theoretical_loss": 3.496949627428438,
+      "objective/train/tokens_used": 125316576,
+      "theoretical_loss": 3.496949627428438,
+      "tokens_seen": 1754857472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.459155833734553e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.496949627428438,
+      "tokens_seen": 1754857472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.45835339431873e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.4969274688276153,
+      "tokens_seen": 1754988544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.457550954902905e-05,
+      "loss": 1.1882,
+      "theoretical_loss": 3.4969053123449942,
+      "tokens_seen": 1755119616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.456748515487082e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.4968831579802133,
+      "tokens_seen": 1755250688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.455946076071257e-05,
+      "loss": 1.1822,
+      "theoretical_loss": 3.496861005732912,
+      "tokens_seen": 1755381760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.455143636655432e-05,
+      "loss": 1.168,
+      "theoretical_loss": 3.4968388556027303,
+      "tokens_seen": 1755512832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.454341197239609e-05,
+      "loss": 1.2653,
+      "theoretical_loss": 3.4968167075893075,
+      "tokens_seen": 1755643904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.453538757823784e-05,
+      "loss": 1.1721,
+      "theoretical_loss": 3.4967945616922838,
+      "tokens_seen": 1755774976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.452736318407961e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.4967724179112984,
+      "tokens_seen": 1755906048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.451933878992136e-05,
+      "loss": 1.1734,
+      "theoretical_loss": 3.496750276245991,
+      "tokens_seen": 1756037120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.451131439576313e-05,
+      "loss": 1.1863,
+      "theoretical_loss": 3.4967281366960026,
+      "tokens_seen": 1756168192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.450329000160488e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.496705999260972,
+      "tokens_seen": 1756299264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.449526560744664e-05,
+      "loss": 1.1681,
+      "theoretical_loss": 3.4966838639405404,
+      "tokens_seen": 1756430336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.44872412132884e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.4966617307343477,
+      "tokens_seen": 1756561408
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.447921681913016e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.496639599642034,
+      "tokens_seen": 1756692480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.447119242497192e-05,
+      "loss": 1.1673,
+      "theoretical_loss": 3.4966174706632396,
+      "tokens_seen": 1756823552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.446316803081368e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.4965953437976056,
+      "tokens_seen": 1756954624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.445514363665544e-05,
+      "loss": 1.133,
+      "theoretical_loss": 3.4965732190447723,
+      "tokens_seen": 1757085696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 9.44471192424972e-05,
+      "loss": 1.1538,
+      "theoretical_loss": 3.4965510964043798,
+      "tokens_seen": 1757216768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.443909484833896e-05,
+      "loss": 1.1405,
+      "theoretical_loss": 3.4965289758760694,
+      "tokens_seen": 1757347840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.443107045418072e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.496506857459482,
+      "tokens_seen": 1757478912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.442304606002247e-05,
+      "loss": 1.1698,
+      "theoretical_loss": 3.4964847411542586,
+      "tokens_seen": 1757609984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.441502166586424e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.49646262696004,
+      "tokens_seen": 1757741056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.440699727170599e-05,
+      "loss": 1.1421,
+      "theoretical_loss": 3.4964405148764675,
+      "tokens_seen": 1757872128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.439897287754776e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.496418404903182,
+      "tokens_seen": 1758003200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.439094848338951e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4963962970398246,
+      "tokens_seen": 1758134272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.438292408923127e-05,
+      "loss": 1.1886,
+      "theoretical_loss": 3.4963741912860375,
+      "tokens_seen": 1758265344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.437489969507303e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.496352087641462,
+      "tokens_seen": 1758396416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.436687530091478e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.4963299861057386,
+      "tokens_seen": 1758527488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.435885090675655e-05,
+      "loss": 1.1886,
+      "theoretical_loss": 3.49630788667851,
+      "tokens_seen": 1758658560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.43508265125983e-05,
+      "loss": 1.209,
+      "theoretical_loss": 3.4962857893594177,
+      "tokens_seen": 1758789632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.434280211844007e-05,
+      "loss": 1.1606,
+      "theoretical_loss": 3.4962636941481033,
+      "tokens_seen": 1758920704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.433477772428182e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.496241601044209,
+      "tokens_seen": 1759051776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.432675333012359e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.4962195100473767,
+      "tokens_seen": 1759182848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.431872893596534e-05,
+      "loss": 1.2904,
+      "theoretical_loss": 3.496197421157248,
+      "tokens_seen": 1759313920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.43107045418071e-05,
+      "loss": 1.1526,
+      "theoretical_loss": 3.496175334373466,
+      "tokens_seen": 1759444992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.430268014764886e-05,
+      "loss": 1.172,
+      "theoretical_loss": 3.4961532496956718,
+      "tokens_seen": 1759576064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.429465575349061e-05,
+      "loss": 1.1458,
+      "theoretical_loss": 3.496131167123509,
+      "tokens_seen": 1759707136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.428663135933238e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.4961090866566185,
+      "tokens_seen": 1759838208
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.427860696517413e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.4960870082946442,
+      "tokens_seen": 1759969280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.42705825710159e-05,
+      "loss": 1.1559,
+      "theoretical_loss": 3.4960649320372283,
+      "tokens_seen": 1760100352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.426255817685765e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.4960428578840137,
+      "tokens_seen": 1760231424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.42545337826994e-05,
+      "loss": 1.1509,
+      "theoretical_loss": 3.4960207858346424,
+      "tokens_seen": 1760362496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.424650938854117e-05,
+      "loss": 1.1957,
+      "theoretical_loss": 3.4959987158887573,
+      "tokens_seen": 1760493568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.423848499438293e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.4959766480460024,
+      "tokens_seen": 1760624640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.423046060022469e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.4959545823060196,
+      "tokens_seen": 1760755712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.422243620606645e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.4959325186684533,
+      "tokens_seen": 1760886784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.421441181190821e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.495910457132945,
+      "tokens_seen": 1761017856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.420638741774997e-05,
+      "loss": 1.1617,
+      "theoretical_loss": 3.4958883976991397,
+      "tokens_seen": 1761148928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.419836302359172e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.49586634036668,
+      "tokens_seen": 1761280000
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 593108,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1524906158447266,
+      "objective/train/theoretical_loss": 3.4958442851352087,
+      "objective/train/tokens_used": 131870176,
+      "theoretical_loss": 3.4958442851352087,
+      "tokens_seen": 1761411072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.419033862943349e-05,
+      "loss": 1.1579,
+      "theoretical_loss": 3.4958442851352087,
+      "tokens_seen": 1761411072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.418231423527524e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.495822232004371,
+      "tokens_seen": 1761542144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.4174289841117e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.495800180973809,
+      "tokens_seen": 1761673216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.416626544695876e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.495778132043167,
+      "tokens_seen": 1761804288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.415824105280053e-05,
+      "loss": 1.2184,
+      "theoretical_loss": 3.495756085212089,
+      "tokens_seen": 1761935360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.415021665864228e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.495734040480219,
+      "tokens_seen": 1762066432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.414219226448404e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.4957119978472,
+      "tokens_seen": 1762197504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.41341678703258e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.4956899573126776,
+      "tokens_seen": 1762328576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.412614347616755e-05,
+      "loss": 1.1593,
+      "theoretical_loss": 3.4956679188762947,
+      "tokens_seen": 1762459648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.411811908200932e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.495645882537696,
+      "tokens_seen": 1762590720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.411009468785107e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.4956238482965265,
+      "tokens_seen": 1762721792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.410207029369284e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.49560181615243,
+      "tokens_seen": 1762852864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.409404589953459e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.4955797861050506,
+      "tokens_seen": 1762983936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.408602150537636e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.495557758154033,
+      "tokens_seen": 1763115008
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.407799711121811e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4955357322990226,
+      "tokens_seen": 1763246080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.406997271705986e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.495513708539664,
+      "tokens_seen": 1763377152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.406194832290163e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.495491686875602,
+      "tokens_seen": 1763508224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.405392392874338e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.495469667306481,
+      "tokens_seen": 1763639296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.404589953458515e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.495447649831947,
+      "tokens_seen": 1763770368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.40378751404269e-05,
+      "loss": 1.1659,
+      "theoretical_loss": 3.495425634451644,
+      "tokens_seen": 1763901440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.402985074626867e-05,
+      "loss": 1.1836,
+      "theoretical_loss": 3.495403621165218,
+      "tokens_seen": 1764032512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.402182635211042e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.495381609972314,
+      "tokens_seen": 1764163584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.401380195795218e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.4953596008725776,
+      "tokens_seen": 1764294656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.400577756379394e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4953375938656537,
+      "tokens_seen": 1764425728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.39977531696357e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.495315588951189,
+      "tokens_seen": 1764556800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.398972877547746e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.495293586128828,
+      "tokens_seen": 1764687872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.398170438131922e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.495271585398217,
+      "tokens_seen": 1764818944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.397367998716098e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.495249586759001,
+      "tokens_seen": 1764950016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.396565559300274e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.495227590210827,
+      "tokens_seen": 1765081088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.395763119884449e-05,
+      "loss": 1.2861,
+      "theoretical_loss": 3.4952055957533408,
+      "tokens_seen": 1765212160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.394960680468626e-05,
+      "loss": 1.1522,
+      "theoretical_loss": 3.495183603386188,
+      "tokens_seen": 1765343232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.394158241052801e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.4951616131090146,
+      "tokens_seen": 1765474304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.393355801636978e-05,
+      "loss": 1.152,
+      "theoretical_loss": 3.495139624921468,
+      "tokens_seen": 1765605376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.392553362221153e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.495117638823193,
+      "tokens_seen": 1765736448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.39175092280533e-05,
+      "loss": 1.1822,
+      "theoretical_loss": 3.495095654813837,
+      "tokens_seen": 1765867520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.390948483389505e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.4950736728930467,
+      "tokens_seen": 1765998592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.39014604397368e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.4950516930604674,
+      "tokens_seen": 1766129664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.389343604557857e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.495029715315747,
+      "tokens_seen": 1766260736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.388541165142032e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.495007739658532,
+      "tokens_seen": 1766391808
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.387738725726209e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.494985766088469,
+      "tokens_seen": 1766522880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.386936286310384e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.494963794605205,
+      "tokens_seen": 1766653952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.386133846894561e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.494941825208387,
+      "tokens_seen": 1766785024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.385331407478736e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.4949198578976626,
+      "tokens_seen": 1766916096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.384528968062911e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.494897892672678,
+      "tokens_seen": 1767047168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.383726528647088e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.494875929533081,
+      "tokens_seen": 1767178240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.382924089231263e-05,
+      "loss": 1.1715,
+      "theoretical_loss": 3.4948539684785196,
+      "tokens_seen": 1767309312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.38212164981544e-05,
+      "loss": 1.1479,
+      "theoretical_loss": 3.4948320095086403,
+      "tokens_seen": 1767440384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.381319210399615e-05,
+      "loss": 1.1902,
+      "theoretical_loss": 3.494810052623091,
+      "tokens_seen": 1767571456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.380516770983792e-05,
+      "loss": 1.1806,
+      "theoretical_loss": 3.494788097821519,
+      "tokens_seen": 1767702528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.379714331567967e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.494766145103573,
+      "tokens_seen": 1767833600
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/docs_used": 595402,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2305282354354858,
+      "objective/train/theoretical_loss": 3.4947441944688995,
+      "objective/train/tokens_used": 138423776,
+      "theoretical_loss": 3.4947441944688995,
+      "tokens_seen": 1767964672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.378911892152144e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.4947441944688995,
+      "tokens_seen": 1767964672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.37810945273632e-05,
+      "loss": 1.2337,
+      "theoretical_loss": 3.494722245917147,
+      "tokens_seen": 1768095744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.377307013320495e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.4947002994479632,
+      "tokens_seen": 1768226816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.376504573904671e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.494678355060997,
+      "tokens_seen": 1768357888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.375702134488847e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.4946564127558952,
+      "tokens_seen": 1768488960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.374899695073023e-05,
+      "loss": 1.1464,
+      "theoretical_loss": 3.494634472532307,
+      "tokens_seen": 1768620032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.374097255657199e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.494612534389881,
+      "tokens_seen": 1768751104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.373294816241375e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.494590598328264,
+      "tokens_seen": 1768882176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.37249237682555e-05,
+      "loss": 1.1747,
+      "theoretical_loss": 3.4945686643471063,
+      "tokens_seen": 1769013248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.371689937409726e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.4945467324460555,
+      "tokens_seen": 1769144320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.370887497993903e-05,
+      "loss": 1.1501,
+      "theoretical_loss": 3.4945248026247597,
+      "tokens_seen": 1769275392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.370085058578078e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.494502874882869,
+      "tokens_seen": 1769406464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.369282619162255e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.4944809492200317,
+      "tokens_seen": 1769537536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.36848017974643e-05,
+      "loss": 1.1404,
+      "theoretical_loss": 3.494459025635896,
+      "tokens_seen": 1769668608
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.367677740330607e-05,
+      "loss": 1.1753,
+      "theoretical_loss": 3.494437104130111,
+      "tokens_seen": 1769799680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.366875300914782e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.4944151847023273,
+      "tokens_seen": 1769930752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.366072861498957e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.4943932673521925,
+      "tokens_seen": 1770061824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.365270422083134e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.494371352079356,
+      "tokens_seen": 1770192896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.364467982667309e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.4943494388834675,
+      "tokens_seen": 1770323968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.363665543251486e-05,
+      "loss": 1.1805,
+      "theoretical_loss": 3.4943275277641765,
+      "tokens_seen": 1770455040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.362863103835661e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.494305618721132,
+      "tokens_seen": 1770586112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.362060664419838e-05,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4942837117539844,
+      "tokens_seen": 1770717184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.361258225004013e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.494261806862383,
+      "tokens_seen": 1770848256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.360455785588188e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.4942399040459766,
+      "tokens_seen": 1770979328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.359653346172365e-05,
+      "loss": 1.1486,
+      "theoretical_loss": 3.4942180033044163,
+      "tokens_seen": 1771110400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.35885090675654e-05,
+      "loss": 1.1753,
+      "theoretical_loss": 3.4941961046373518,
+      "tokens_seen": 1771241472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.358048467340717e-05,
+      "loss": 1.1788,
+      "theoretical_loss": 3.4941742080444325,
+      "tokens_seen": 1771372544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.357246027924892e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.494152313525309,
+      "tokens_seen": 1771503616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.356443588509069e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.494130421079632,
+      "tokens_seen": 1771634688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.355641149093244e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.4941085307070496,
+      "tokens_seen": 1771765760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.35483870967742e-05,
+      "loss": 1.1545,
+      "theoretical_loss": 3.494086642407215,
+      "tokens_seen": 1771896832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.354036270261596e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.494064756179777,
+      "tokens_seen": 1772027904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.353233830845772e-05,
+      "loss": 1.2078,
+      "theoretical_loss": 3.4940428720243863,
+      "tokens_seen": 1772158976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.352431391429948e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.4940209899406933,
+      "tokens_seen": 1772290048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.351628952014124e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.4939991099283496,
+      "tokens_seen": 1772421120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.3508265125983e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.493977231987005,
+      "tokens_seen": 1772552192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.350024073182476e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.493955356116311,
+      "tokens_seen": 1772683264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.349221633766652e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.493933482315918,
+      "tokens_seen": 1772814336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.348419194350828e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.493911610585477,
+      "tokens_seen": 1772945408
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.347616754935003e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.49388974092464,
+      "tokens_seen": 1773076480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.34681431551918e-05,
+      "loss": 1.3383,
+      "theoretical_loss": 3.4938678733330573,
+      "tokens_seen": 1773207552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.346011876103355e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.493846007810381,
+      "tokens_seen": 1773338624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.345209436687532e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.493824144356261,
+      "tokens_seen": 1773469696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.344406997271707e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.4938022829703503,
+      "tokens_seen": 1773600768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 9.343604557855882e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4937804236523,
+      "tokens_seen": 1773731840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.342802118440059e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.493758566401761,
+      "tokens_seen": 1773862912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.341999679024234e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.4937367112183857,
+      "tokens_seen": 1773993984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.341197239608411e-05,
+      "loss": 1.1601,
+      "theoretical_loss": 3.493714858101826,
+      "tokens_seen": 1774125056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.340394800192586e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.493693007051733,
+      "tokens_seen": 1774256128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.339592360776763e-05,
+      "loss": 1.0947,
+      "theoretical_loss": 3.4936711580677597,
+      "tokens_seen": 1774387200
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 597604,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.145444631576538,
+      "objective/train/theoretical_loss": 3.4936493111495572,
+      "objective/train/tokens_used": 144977376,
+      "theoretical_loss": 3.4936493111495572,
+      "tokens_seen": 1774518272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.338789921360938e-05,
+      "loss": 1.1758,
+      "theoretical_loss": 3.4936493111495572,
+      "tokens_seen": 1774518272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.337987481945113e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.493627466296778,
+      "tokens_seen": 1774649344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.33718504252929e-05,
+      "loss": 1.147,
+      "theoretical_loss": 3.4936056235090747,
+      "tokens_seen": 1774780416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.336382603113465e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.493583782786099,
+      "tokens_seen": 1774911488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.335580163697642e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.4935619441275034,
+      "tokens_seen": 1775042560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.334777724281817e-05,
+      "loss": 1.1572,
+      "theoretical_loss": 3.4935401075329406,
+      "tokens_seen": 1775173632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.333975284865993e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.4935182730020626,
+      "tokens_seen": 1775304704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.33317284545017e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.493496440534523,
+      "tokens_seen": 1775435776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.332370406034345e-05,
+      "loss": 1.0931,
+      "theoretical_loss": 3.493474610129973,
+      "tokens_seen": 1775566848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.331567966618521e-05,
+      "loss": 1.2034,
+      "theoretical_loss": 3.4934527817880676,
+      "tokens_seen": 1775697920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.330765527202697e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4934309555084573,
+      "tokens_seen": 1775828992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.329963087786872e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.4934091312907967,
+      "tokens_seen": 1775960064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.329160648371049e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.4933873091347385,
+      "tokens_seen": 1776091136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.328358208955224e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.493365489039935,
+      "tokens_seen": 1776222208
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.3275557695394e-05,
+      "loss": 1.1691,
+      "theoretical_loss": 3.4933436710060404,
+      "tokens_seen": 1776353280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.326753330123576e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4933218550327076,
+      "tokens_seen": 1776484352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.325950890707753e-05,
+      "loss": 1.1749,
+      "theoretical_loss": 3.49330004111959,
+      "tokens_seen": 1776615424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.325148451291928e-05,
+      "loss": 1.1356,
+      "theoretical_loss": 3.4932782292663416,
+      "tokens_seen": 1776746496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.324346011876103e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4932564194726154,
+      "tokens_seen": 1776877568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.32354357246028e-05,
+      "loss": 1.1934,
+      "theoretical_loss": 3.493234611738065,
+      "tokens_seen": 1777008640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.322741133044455e-05,
+      "loss": 1.1536,
+      "theoretical_loss": 3.4932128060623437,
+      "tokens_seen": 1777139712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.321938693628632e-05,
+      "loss": 1.181,
+      "theoretical_loss": 3.493191002445106,
+      "tokens_seen": 1777270784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.321136254212807e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.4931692008860056,
+      "tokens_seen": 1777401856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.320333814796983e-05,
+      "loss": 1.0847,
+      "theoretical_loss": 3.4931474013846966,
+      "tokens_seen": 1777532928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.319531375381159e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.4931256039408334,
+      "tokens_seen": 1777664000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.318728935965335e-05,
+      "loss": 1.1873,
+      "theoretical_loss": 3.493103808554069,
+      "tokens_seen": 1777795072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.317926496549511e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4930820152240587,
+      "tokens_seen": 1777926144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.317124057133686e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4930602239504562,
+      "tokens_seen": 1778057216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.316321617717863e-05,
+      "loss": 1.1135,
+      "theoretical_loss": 3.493038434732916,
+      "tokens_seen": 1778188288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.315519178302038e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.4930166475710926,
+      "tokens_seen": 1778319360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.314716738886214e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.4929948624646414,
+      "tokens_seen": 1778450432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.31391429947039e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.4929730794132157,
+      "tokens_seen": 1778581504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.313111860054566e-05,
+      "loss": 1.1427,
+      "theoretical_loss": 3.492951298416471,
+      "tokens_seen": 1778712576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.312309420638742e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.4929295194740613,
+      "tokens_seen": 1778843648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.311506981222918e-05,
+      "loss": 1.1551,
+      "theoretical_loss": 3.4929077425856425,
+      "tokens_seen": 1778974720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.310704541807093e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.492885967750869,
+      "tokens_seen": 1779105792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.30990210239127e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.492864194969396,
+      "tokens_seen": 1779236864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.309099662975445e-05,
+      "loss": 1.1746,
+      "theoretical_loss": 3.4928424242408784,
+      "tokens_seen": 1779367936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.308297223559622e-05,
+      "loss": 1.279,
+      "theoretical_loss": 3.492820655564972,
+      "tokens_seen": 1779499008
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.307494784143797e-05,
+      "loss": 1.1746,
+      "theoretical_loss": 3.4927988889413317,
+      "tokens_seen": 1779630080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.306692344727974e-05,
+      "loss": 1.1779,
+      "theoretical_loss": 3.4927771243696135,
+      "tokens_seen": 1779761152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.305889905312149e-05,
+      "loss": 1.176,
+      "theoretical_loss": 3.492755361849472,
+      "tokens_seen": 1779892224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.305087465896324e-05,
+      "loss": 1.1818,
+      "theoretical_loss": 3.4927336013805625,
+      "tokens_seen": 1780023296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.304285026480501e-05,
+      "loss": 1.1254,
+      "theoretical_loss": 3.492711842962542,
+      "tokens_seen": 1780154368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.303482587064676e-05,
+      "loss": 1.1806,
+      "theoretical_loss": 3.492690086595065,
+      "tokens_seen": 1780285440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.302680147648853e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.492668332277788,
+      "tokens_seen": 1780416512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.301877708233028e-05,
+      "loss": 1.178,
+      "theoretical_loss": 3.4926465800103665,
+      "tokens_seen": 1780547584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.301075268817204e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.4926248297924567,
+      "tokens_seen": 1780678656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.30027282940138e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.492603081623715,
+      "tokens_seen": 1780809728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.299470389985556e-05,
+      "loss": 1.1866,
+      "theoretical_loss": 3.4925813355037967,
+      "tokens_seen": 1780940800
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 599799,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1599756479263306,
+      "objective/train/theoretical_loss": 3.492559591432358,
+      "objective/train/tokens_used": 151530976,
+      "theoretical_loss": 3.492559591432358,
+      "tokens_seen": 1781071872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.298667950569732e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.492559591432358,
+      "tokens_seen": 1781071872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.297865511153908e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.4925378494090564,
+      "tokens_seen": 1781202944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.297063071738084e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.492516109433547,
+      "tokens_seen": 1781334016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.29626063232226e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4924943715054866,
+      "tokens_seen": 1781465088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.295458192906435e-05,
+      "loss": 1.2113,
+      "theoretical_loss": 3.492472635624533,
+      "tokens_seen": 1781596160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.294655753490612e-05,
+      "loss": 1.1816,
+      "theoretical_loss": 3.492450901790341,
+      "tokens_seen": 1781727232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.293853314074787e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.492429170002568,
+      "tokens_seen": 1781858304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.293050874658963e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.4924074402608714,
+      "tokens_seen": 1781989376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.292248435243139e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.492385712564907,
+      "tokens_seen": 1782120448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.291445995827315e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.4923639869143326,
+      "tokens_seen": 1782251520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.290643556411491e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4923422633088053,
+      "tokens_seen": 1782382592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.289841116995666e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.492320541747982,
+      "tokens_seen": 1782513664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.289038677579843e-05,
+      "loss": 1.3074,
+      "theoretical_loss": 3.492298822231519,
+      "tokens_seen": 1782644736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.288236238164018e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.492277104759075,
+      "tokens_seen": 1782775808
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.287433798748195e-05,
+      "loss": 1.1876,
+      "theoretical_loss": 3.4922553893303068,
+      "tokens_seen": 1782906880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.28663135933237e-05,
+      "loss": 1.3238,
+      "theoretical_loss": 3.492233675944872,
+      "tokens_seen": 1783037952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.285828919916547e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4922119646024283,
+      "tokens_seen": 1783169024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.285026480500722e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.4921902553026327,
+      "tokens_seen": 1783300096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.284224041084899e-05,
+      "loss": 1.2241,
+      "theoretical_loss": 3.492168548045143,
+      "tokens_seen": 1783431168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.283421601669074e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.4921468428296176,
+      "tokens_seen": 1783562240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.28261916225325e-05,
+      "loss": 1.2768,
+      "theoretical_loss": 3.4921251396557142,
+      "tokens_seen": 1783693312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.281816722837426e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.4921034385230905,
+      "tokens_seen": 1783824384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.281014283421601e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.4920817394314048,
+      "tokens_seen": 1783955456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.280211844005778e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4920600423803148,
+      "tokens_seen": 1784086528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.279409404589953e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.492038347369479,
+      "tokens_seen": 1784217600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.27860696517413e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.4920166543985554,
+      "tokens_seen": 1784348672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.277804525758305e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.491994963467203,
+      "tokens_seen": 1784479744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.27700208634248e-05,
+      "loss": 1.3129,
+      "theoretical_loss": 3.4919732745750798,
+      "tokens_seen": 1784610816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.276199646926657e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4919515877218443,
+      "tokens_seen": 1784741888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.275397207510833e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.491929902907155,
+      "tokens_seen": 1784872960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.274594768095009e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4919082201306715,
+      "tokens_seen": 1785004032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.273792328679185e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.491886539392052,
+      "tokens_seen": 1785135104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.272989889263361e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4918648606909546,
+      "tokens_seen": 1785266176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.272187449847537e-05,
+      "loss": 1.2996,
+      "theoretical_loss": 3.491843184027039,
+      "tokens_seen": 1785397248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.271385010431712e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4918215093999643,
+      "tokens_seen": 1785528320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.270582571015889e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.4917998368093897,
+      "tokens_seen": 1785659392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.269780131600064e-05,
+      "loss": 1.2859,
+      "theoretical_loss": 3.4917781662549734,
+      "tokens_seen": 1785790464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.26897769218424e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.491756497736376,
+      "tokens_seen": 1785921536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.268175252768416e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.4917348312532566,
+      "tokens_seen": 1786052608
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.267372813352592e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.491713166805274,
+      "tokens_seen": 1786183680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.266570373936768e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.4916915043920875,
+      "tokens_seen": 1786314752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.265767934520943e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.491669844013358,
+      "tokens_seen": 1786445824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.26496549510512e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.491648185668744,
+      "tokens_seen": 1786576896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.264163055689295e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.491626529357906,
+      "tokens_seen": 1786707968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.263360616273472e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.4916048750805033,
+      "tokens_seen": 1786839040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.262558176857647e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.491583222836196,
+      "tokens_seen": 1786970112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.261755737441824e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4915615726246445,
+      "tokens_seen": 1787101184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.260953298025999e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.4915399244455085,
+      "tokens_seen": 1787232256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.260150858610174e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.491518278298448,
+      "tokens_seen": 1787363328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.259348419194351e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4914966341831235,
+      "tokens_seen": 1787494400
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/docs_used": 601994,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3068506717681885,
+      "objective/train/theoretical_loss": 3.491474992099196,
+      "objective/train/tokens_used": 158084576,
+      "theoretical_loss": 3.491474992099196,
+      "tokens_seen": 1787625472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.258545979778526e-05,
+      "loss": 1.2604,
+      "theoretical_loss": 3.491474992099196,
+      "tokens_seen": 1787625472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.257743540362703e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.4914533520463245,
+      "tokens_seen": 1787756544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.256941100946878e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.49143171402417,
+      "tokens_seen": 1787887616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.256138661531055e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.491410078032394,
+      "tokens_seen": 1788018688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.25533622211523e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.4913884440706564,
+      "tokens_seen": 1788149760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.254533782699407e-05,
+      "loss": 1.1754,
+      "theoretical_loss": 3.491366812138618,
+      "tokens_seen": 1788280832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.253731343283582e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4913451822359396,
+      "tokens_seen": 1788411904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.252928903867758e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.4913235543622823,
+      "tokens_seen": 1788542976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.252126464451934e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.4913019285173066,
+      "tokens_seen": 1788674048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.25132402503611e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.491280304700674,
+      "tokens_seen": 1788805120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.250521585620286e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.491258682912046,
+      "tokens_seen": 1788936192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.249719146204462e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.4912370631510834,
+      "tokens_seen": 1789067264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.248916706788638e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4912154454174478,
+      "tokens_seen": 1789198336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.248114267372814e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4911938297108,
+      "tokens_seen": 1789329408
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.247311827956989e-05,
+      "loss": 1.3068,
+      "theoretical_loss": 3.491172216030802,
+      "tokens_seen": 1789460480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.246509388541166e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4911506043771148,
+      "tokens_seen": 1789591552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.245706949125341e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.491128994749401,
+      "tokens_seen": 1789722624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.244904509709518e-05,
+      "loss": 1.3005,
+      "theoretical_loss": 3.491107387147321,
+      "tokens_seen": 1789853696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.244102070293693e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.491085781570538,
+      "tokens_seen": 1789984768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 9.24329963087787e-05,
+      "loss": 1.1703,
+      "theoretical_loss": 3.4910641780187133,
+      "tokens_seen": 1790115840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.242497191462045e-05,
+      "loss": 1.2963,
+      "theoretical_loss": 3.491042576491508,
+      "tokens_seen": 1790246912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.24169475204622e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.491020976988586,
+      "tokens_seen": 1790377984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.240892312630397e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.4909993795096077,
+      "tokens_seen": 1790509056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.240089873214572e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.490977784054236,
+      "tokens_seen": 1790640128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.239287433798749e-05,
+      "loss": 1.309,
+      "theoretical_loss": 3.490956190622134,
+      "tokens_seen": 1790771200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.238484994382924e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.490934599212962,
+      "tokens_seen": 1790902272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.237682554967101e-05,
+      "loss": 1.1828,
+      "theoretical_loss": 3.490913009826385,
+      "tokens_seen": 1791033344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.236880115551276e-05,
+      "loss": 1.1898,
+      "theoretical_loss": 3.4908914224620635,
+      "tokens_seen": 1791164416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.236077676135451e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4908698371196607,
+      "tokens_seen": 1791295488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.235275236719628e-05,
+      "loss": 1.2878,
+      "theoretical_loss": 3.4908482537988403,
+      "tokens_seen": 1791426560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.234472797303803e-05,
+      "loss": 1.1384,
+      "theoretical_loss": 3.4908266724992636,
+      "tokens_seen": 1791557632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.23367035788798e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.4908050932205943,
+      "tokens_seen": 1791688704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.232867918472155e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.490783515962495,
+      "tokens_seen": 1791819776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.232065479056332e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.490761940724629,
+      "tokens_seen": 1791950848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.231263039640507e-05,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4907403675066595,
+      "tokens_seen": 1792081920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.230460600224683e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.490718796308249,
+      "tokens_seen": 1792212992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.229658160808859e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4906972271290613,
+      "tokens_seen": 1792344064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.228855721393035e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.4906756599687596,
+      "tokens_seen": 1792475136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.228053281977211e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.4906540948270077,
+      "tokens_seen": 1792606208
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.227250842561387e-05,
+      "loss": 1.2722,
+      "theoretical_loss": 3.4906325317034685,
+      "tokens_seen": 1792737280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.226448403145563e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.490610970597806,
+      "tokens_seen": 1792868352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.225645963729739e-05,
+      "loss": 1.1719,
+      "theoretical_loss": 3.490589411509683,
+      "tokens_seen": 1792999424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.224843524313914e-05,
+      "loss": 1.277,
+      "theoretical_loss": 3.490567854438765,
+      "tokens_seen": 1793130496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.22404108489809e-05,
+      "loss": 1.2127,
+      "theoretical_loss": 3.4905462993847145,
+      "tokens_seen": 1793261568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.223238645482266e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.4905247463471953,
+      "tokens_seen": 1793392640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.222436206066443e-05,
+      "loss": 1.2887,
+      "theoretical_loss": 3.4905031953258723,
+      "tokens_seen": 1793523712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.221633766650618e-05,
+      "loss": 1.3283,
+      "theoretical_loss": 3.490481646320409,
+      "tokens_seen": 1793654784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.220831327234795e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.4904600993304697,
+      "tokens_seen": 1793785856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.22002888781897e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.4904385543557184,
+      "tokens_seen": 1793916928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.219226448403146e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.4904170113958193,
+      "tokens_seen": 1794048000
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 604097,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1891400814056396,
+      "objective/train/theoretical_loss": 3.4903954704504376,
+      "objective/train/tokens_used": 164638176,
+      "theoretical_loss": 3.4903954704504376,
+      "tokens_seen": 1794179072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.218424008987322e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.4903954704504376,
+      "tokens_seen": 1794179072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.217621569571497e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.490373931519237,
+      "tokens_seen": 1794310144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.216819130155674e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.4903523946018824,
+      "tokens_seen": 1794441216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.216016690739849e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.490330859698038,
+      "tokens_seen": 1794572288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.215214251324026e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.4903093268073686,
+      "tokens_seen": 1794703360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.214411811908201e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4902877959295395,
+      "tokens_seen": 1794834432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.213609372492378e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.4902662670642157,
+      "tokens_seen": 1794965504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.212806933076553e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.490244740211061,
+      "tokens_seen": 1795096576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.212004493660728e-05,
+      "loss": 1.3216,
+      "theoretical_loss": 3.490223215369742,
+      "tokens_seen": 1795227648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.211202054244905e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.490201692539922,
+      "tokens_seen": 1795358720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.21039961482908e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.490180171721268,
+      "tokens_seen": 1795489792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.209597175413257e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.4901586529134443,
+      "tokens_seen": 1795620864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.208794735997432e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.490137136116116,
+      "tokens_seen": 1795751936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.207992296581609e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.4901156213289495,
+      "tokens_seen": 1795883008
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.207189857165784e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.4900941085516095,
+      "tokens_seen": 1796014080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.20638741774996e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.4900725977837617,
+      "tokens_seen": 1796145152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.205584978334136e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.490051089025072,
+      "tokens_seen": 1796276224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.204782538918312e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.490029582275206,
+      "tokens_seen": 1796407296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.203980099502488e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.4900080775338296,
+      "tokens_seen": 1796538368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.203177660086664e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.489986574800609,
+      "tokens_seen": 1796669440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.20237522067084e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.4899650740752097,
+      "tokens_seen": 1796800512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.201572781255016e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.489943575357298,
+      "tokens_seen": 1796931584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.200770341839191e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4899220786465404,
+      "tokens_seen": 1797062656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.199967902423368e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4899005839426023,
+      "tokens_seen": 1797193728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.199165463007543e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.489879091245151,
+      "tokens_seen": 1797324800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.19836302359172e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4898576005538517,
+      "tokens_seen": 1797455872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.197560584175895e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.4898361118683723,
+      "tokens_seen": 1797586944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.196758144760072e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4898146251883784,
+      "tokens_seen": 1797718016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.195955705344247e-05,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4897931405135365,
+      "tokens_seen": 1797849088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.195153265928422e-05,
+      "loss": 1.3082,
+      "theoretical_loss": 3.489771657843514,
+      "tokens_seen": 1797980160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.194350826512599e-05,
+      "loss": 1.2643,
+      "theoretical_loss": 3.489750177177977,
+      "tokens_seen": 1798111232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.193548387096774e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.4897286985165934,
+      "tokens_seen": 1798242304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.192745947680951e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.489707221859029,
+      "tokens_seen": 1798373376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.191943508265126e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4896857472049514,
+      "tokens_seen": 1798504448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.191141068849303e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.489664274554028,
+      "tokens_seen": 1798635520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.190338629433478e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.489642803905925,
+      "tokens_seen": 1798766592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.189536190017655e-05,
+      "loss": 1.1528,
+      "theoretical_loss": 3.4896213352603107,
+      "tokens_seen": 1798897664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.18873375060183e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.4895998686168515,
+      "tokens_seen": 1799028736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.187931311186005e-05,
+      "loss": 1.1284,
+      "theoretical_loss": 3.489578403975216,
+      "tokens_seen": 1799159808
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.187128871770182e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.4895569413350707,
+      "tokens_seen": 1799290880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.186326432354357e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.489535480696084,
+      "tokens_seen": 1799421952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.185523992938534e-05,
+      "loss": 1.2127,
+      "theoretical_loss": 3.4895140220579224,
+      "tokens_seen": 1799553024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.18472155352271e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.489492565420255,
+      "tokens_seen": 1799684096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.183919114106886e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.489471110782749,
+      "tokens_seen": 1799815168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.183116674691061e-05,
+      "loss": 1.3249,
+      "theoretical_loss": 3.489449658145072,
+      "tokens_seen": 1799946240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.182314235275237e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.4894282075068923,
+      "tokens_seen": 1800077312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.181511795859413e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.489406758867878,
+      "tokens_seen": 1800208384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.180709356443589e-05,
+      "loss": 1.1699,
+      "theoretical_loss": 3.4893853122276974,
+      "tokens_seen": 1800339456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.179906917027765e-05,
+      "loss": 1.3182,
+      "theoretical_loss": 3.489363867586018,
+      "tokens_seen": 1800470528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.17910447761194e-05,
+      "loss": 1.1858,
+      "theoretical_loss": 3.4893424249425093,
+      "tokens_seen": 1800601600
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/docs_used": 606247,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2453995943069458,
+      "objective/train/theoretical_loss": 3.489320984296839,
+      "objective/train/tokens_used": 171191776,
+      "theoretical_loss": 3.489320984296839,
+      "tokens_seen": 1800732672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.178302038196117e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.489320984296839,
+      "tokens_seen": 1800732672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.177499598780293e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.4892995456486746,
+      "tokens_seen": 1800863744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.176697159364468e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.4892781089976865,
+      "tokens_seen": 1800994816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.175894719948645e-05,
+      "loss": 1.1797,
+      "theoretical_loss": 3.489256674343542,
+      "tokens_seen": 1801125888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.17509228053282e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4892352416859107,
+      "tokens_seen": 1801256960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.174289841116997e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.4892138110244604,
+      "tokens_seen": 1801388032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.173487401701172e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.489192382358861,
+      "tokens_seen": 1801519104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.172684962285349e-05,
+      "loss": 1.3103,
+      "theoretical_loss": 3.48917095568878,
+      "tokens_seen": 1801650176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.171882522869524e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.4891495310138883,
+      "tokens_seen": 1801781248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.171080083453699e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.489128108333854,
+      "tokens_seen": 1801912320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.170277644037876e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.4891066876483454,
+      "tokens_seen": 1802043392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.169475204622051e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.489085268957033,
+      "tokens_seen": 1802174464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.168672765206228e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.489063852259586,
+      "tokens_seen": 1802305536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.167870325790403e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.489042437555673,
+      "tokens_seen": 1802436608
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.16706788637458e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4890210248449645,
+      "tokens_seen": 1802567680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.166265446958755e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.4889996141271293,
+      "tokens_seen": 1802698752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.16546300754293e-05,
+      "loss": 1.316,
+      "theoretical_loss": 3.4889782054018377,
+      "tokens_seen": 1802829824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.164660568127107e-05,
+      "loss": 1.1919,
+      "theoretical_loss": 3.4889567986687586,
+      "tokens_seen": 1802960896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.163858128711282e-05,
+      "loss": 1.3322,
+      "theoretical_loss": 3.488935393927563,
+      "tokens_seen": 1803091968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.163055689295459e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4889139911779194,
+      "tokens_seen": 1803223040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.162253249879634e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.488892590419499,
+      "tokens_seen": 1803354112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.161450810463811e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.488871191651971,
+      "tokens_seen": 1803485184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.160648371047986e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.4888497948750055,
+      "tokens_seen": 1803616256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.159845931632163e-05,
+      "loss": 1.3276,
+      "theoretical_loss": 3.4888284000882734,
+      "tokens_seen": 1803747328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.159043492216338e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.4888070072914443,
+      "tokens_seen": 1803878400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.158241052800514e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.4887856164841886,
+      "tokens_seen": 1804009472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.15743861338469e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.488764227666177,
+      "tokens_seen": 1804140544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.156636173968866e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.48874284083708,
+      "tokens_seen": 1804271616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.155833734553042e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.4887214559965685,
+      "tokens_seen": 1804402688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.155031295137218e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4887000731443125,
+      "tokens_seen": 1804533760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.154228855721394e-05,
+      "loss": 1.3469,
+      "theoretical_loss": 3.488678692279983,
+      "tokens_seen": 1804664832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.15342641630557e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.488657313403251,
+      "tokens_seen": 1804795904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.152623976889745e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.488635936513787,
+      "tokens_seen": 1804926976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.151821537473922e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4886145616112625,
+      "tokens_seen": 1805058048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.151019098058097e-05,
+      "loss": 1.2982,
+      "theoretical_loss": 3.488593188695348,
+      "tokens_seen": 1805189120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.150216658642274e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4885718177657155,
+      "tokens_seen": 1805320192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.149414219226449e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.488550448822035,
+      "tokens_seen": 1805451264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.148611779810626e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.488529081863979,
+      "tokens_seen": 1805582336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.147809340394801e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.4885077168912177,
+      "tokens_seen": 1805713408
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.147006900978976e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.488486353903424,
+      "tokens_seen": 1805844480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.146204461563153e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.4884649929002682,
+      "tokens_seen": 1805975552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.145402022147328e-05,
+      "loss": 1.3126,
+      "theoretical_loss": 3.488443633881422,
+      "tokens_seen": 1806106624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.144599582731505e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.488422276846558,
+      "tokens_seen": 1806237696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.14379714331568e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.488400921795347,
+      "tokens_seen": 1806368768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.142994703899857e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.488379568727461,
+      "tokens_seen": 1806499840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 9.142192264484032e-05,
+      "loss": 1.3171,
+      "theoretical_loss": 3.4883582176425723,
+      "tokens_seen": 1806630912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.141389825068207e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.488336868540353,
+      "tokens_seen": 1806761984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.140587385652384e-05,
+      "loss": 1.162,
+      "theoretical_loss": 3.488315521420475,
+      "tokens_seen": 1806893056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.13978494623656e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4882941762826096,
+      "tokens_seen": 1807024128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.138982506820736e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.4882728331264303,
+      "tokens_seen": 1807155200
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 608509,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.333984136581421,
+      "objective/train/theoretical_loss": 3.4882514919516088,
+      "objective/train/tokens_used": 177745376,
+      "theoretical_loss": 3.4882514919516088,
+      "tokens_seen": 1807286272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.138180067404911e-05,
+      "loss": 1.3397,
+      "theoretical_loss": 3.4882514919516088,
+      "tokens_seen": 1807286272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.137377627989088e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4882301527578177,
+      "tokens_seen": 1807417344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.136575188573263e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.488208815544729,
+      "tokens_seen": 1807548416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.135772749157439e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.488187480312016,
+      "tokens_seen": 1807679488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.134970309741615e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.48816614705935,
+      "tokens_seen": 1807810560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.13416787032579e-05,
+      "loss": 1.3049,
+      "theoretical_loss": 3.4881448157864057,
+      "tokens_seen": 1807941632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.133365430909967e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.4881234864928543,
+      "tokens_seen": 1808072704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.132562991494143e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.488102159178369,
+      "tokens_seen": 1808203776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.131760552078319e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.4880808338426235,
+      "tokens_seen": 1808334848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.130958112662495e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4880595104852894,
+      "tokens_seen": 1808465920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.13015567324667e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.488038189106041,
+      "tokens_seen": 1808596992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.129353233830847e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.488016869704551,
+      "tokens_seen": 1808728064
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.128550794415022e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4879955522804926,
+      "tokens_seen": 1808859136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.127748354999199e-05,
+      "loss": 1.316,
+      "theoretical_loss": 3.487974236833539,
+      "tokens_seen": 1808990208
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.126945915583374e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.4879529233633644,
+      "tokens_seen": 1809121280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.12614347616755e-05,
+      "loss": 1.2813,
+      "theoretical_loss": 3.487931611869641,
+      "tokens_seen": 1809252352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.125341036751726e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4879103023520432,
+      "tokens_seen": 1809383424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.124538597335903e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.4878889948102447,
+      "tokens_seen": 1809514496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.123736157920078e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.4878676892439184,
+      "tokens_seen": 1809645568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.122933718504253e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4878463856527393,
+      "tokens_seen": 1809776640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.12213127908843e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.4878250840363805,
+      "tokens_seen": 1809907712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.121328839672605e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.487803784394516,
+      "tokens_seen": 1810038784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.120526400256782e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4877824867268195,
+      "tokens_seen": 1810169856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.119723960840957e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.4877611910329662,
+      "tokens_seen": 1810300928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.118921521425134e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.487739897312629,
+      "tokens_seen": 1810432000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.118119082009309e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.4877186055654823,
+      "tokens_seen": 1810563072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.117316642593484e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.487697315791202,
+      "tokens_seen": 1810694144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.116514203177661e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.48767602798946,
+      "tokens_seen": 1810825216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.115711763761836e-05,
+      "loss": 1.2906,
+      "theoretical_loss": 3.4876547421599327,
+      "tokens_seen": 1810956288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.114909324346013e-05,
+      "loss": 1.2985,
+      "theoretical_loss": 3.4876334583022945,
+      "tokens_seen": 1811087360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.114106884930188e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.487612176416219,
+      "tokens_seen": 1811218432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.113304445514365e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.4875908965013815,
+      "tokens_seen": 1811349504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.11250200609854e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.4875696185574574,
+      "tokens_seen": 1811480576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.111699566682716e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.4875483425841205,
+      "tokens_seen": 1811611648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.110897127266892e-05,
+      "loss": 1.3242,
+      "theoretical_loss": 3.4875270685810467,
+      "tokens_seen": 1811742720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.110094687851068e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.48750579654791,
+      "tokens_seen": 1811873792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.109292248435244e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.487484526484386,
+      "tokens_seen": 1812004864
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.10848980901942e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.48746325839015,
+      "tokens_seen": 1812135936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.107687369603596e-05,
+      "loss": 1.3488,
+      "theoretical_loss": 3.487441992264878,
+      "tokens_seen": 1812267008
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.106884930187772e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.4874207281082437,
+      "tokens_seen": 1812398080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.106082490771947e-05,
+      "loss": 1.1907,
+      "theoretical_loss": 3.487399465919924,
+      "tokens_seen": 1812529152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.105280051356124e-05,
+      "loss": 1.3253,
+      "theoretical_loss": 3.487378205699593,
+      "tokens_seen": 1812660224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.104477611940299e-05,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4873569474469273,
+      "tokens_seen": 1812791296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.103675172524476e-05,
+      "loss": 1.2979,
+      "theoretical_loss": 3.4873356911616016,
+      "tokens_seen": 1812922368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.102872733108651e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4873144368432927,
+      "tokens_seen": 1813053440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.102070293692828e-05,
+      "loss": 1.1789,
+      "theoretical_loss": 3.4872931844916764,
+      "tokens_seen": 1813184512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.101267854277003e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4872719341064276,
+      "tokens_seen": 1813315584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.100465414861178e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.487250685687223,
+      "tokens_seen": 1813446656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.099662975445355e-05,
+      "loss": 1.288,
+      "theoretical_loss": 3.487229439233738,
+      "tokens_seen": 1813577728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.09886053602953e-05,
+      "loss": 1.1914,
+      "theoretical_loss": 3.4872081947456492,
+      "tokens_seen": 1813708800
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 610938,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2604708671569824,
+      "objective/train/theoretical_loss": 3.4871869522226326,
+      "objective/train/tokens_used": 184298976,
+      "theoretical_loss": 3.4871869522226326,
+      "tokens_seen": 1813839872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.098058096613707e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.4871869522226326,
+      "tokens_seen": 1813839872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.097255657197882e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.487165711664365,
+      "tokens_seen": 1813970944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.096453217782059e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4871444730705212,
+      "tokens_seen": 1814102016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.095650778366234e-05,
+      "loss": 1.3299,
+      "theoretical_loss": 3.4871232364407794,
+      "tokens_seen": 1814233088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.094848338950411e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.487102001774815,
+      "tokens_seen": 1814364160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.094045899534586e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.487080769072305,
+      "tokens_seen": 1814495232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.093243460118761e-05,
+      "loss": 1.1776,
+      "theoretical_loss": 3.487059538332926,
+      "tokens_seen": 1814626304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.092441020702938e-05,
+      "loss": 1.2338,
+      "theoretical_loss": 3.4870383095563544,
+      "tokens_seen": 1814757376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.091638581287113e-05,
+      "loss": 1.3278,
+      "theoretical_loss": 3.4870170827422675,
+      "tokens_seen": 1814888448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.09083614187129e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.486995857890342,
+      "tokens_seen": 1815019520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.090033702455465e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.486974635000254,
+      "tokens_seen": 1815150592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.089231263039642e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.486953414071682,
+      "tokens_seen": 1815281664
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.088428823623817e-05,
+      "loss": 1.2837,
+      "theoretical_loss": 3.486932195104302,
+      "tokens_seen": 1815412736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.087626384207993e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.486910978097792,
+      "tokens_seen": 1815543808
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.08682394479217e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.4868897630518285,
+      "tokens_seen": 1815674880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.086021505376345e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.486868549966089,
+      "tokens_seen": 1815805952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.085219065960521e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.4868473388402506,
+      "tokens_seen": 1815937024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.084416626544697e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.4868261296739917,
+      "tokens_seen": 1816068096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.083614187128873e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.4868049224669893,
+      "tokens_seen": 1816199168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.082811747713049e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.4867837172189207,
+      "tokens_seen": 1816330240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.082009308297224e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.486762513929464,
+      "tokens_seen": 1816461312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.0812068688814e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.486741312598298,
+      "tokens_seen": 1816592384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.080404429465576e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.4867201132250982,
+      "tokens_seen": 1816723456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.079601990049753e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.4866989158095443,
+      "tokens_seen": 1816854528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.078799550633928e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.486677720351314,
+      "tokens_seen": 1816985600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.077997111218105e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.4866565268500853,
+      "tokens_seen": 1817116672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.07719467180228e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.486635335305536,
+      "tokens_seen": 1817247744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.076392232386455e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.486614145717345,
+      "tokens_seen": 1817378816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.075589792970632e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.48659295808519,
+      "tokens_seen": 1817509888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.074787353554807e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.48657177240875,
+      "tokens_seen": 1817640960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.073984914138984e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4865505886877033,
+      "tokens_seen": 1817772032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.073182474723159e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4865294069217274,
+      "tokens_seen": 1817903104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.072380035307334e-05,
+      "loss": 1.2813,
+      "theoretical_loss": 3.4865082271105026,
+      "tokens_seen": 1818034176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.071577595891511e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.4864870492537063,
+      "tokens_seen": 1818165248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.070775156475686e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.486465873351018,
+      "tokens_seen": 1818296320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.069972717059863e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.4864446994021168,
+      "tokens_seen": 1818427392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.069170277644038e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.4864235274066804,
+      "tokens_seen": 1818558464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.068367838228215e-05,
+      "loss": 1.2828,
+      "theoretical_loss": 3.486402357364389,
+      "tokens_seen": 1818689536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.06756539881239e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.4863811892749212,
+      "tokens_seen": 1818820608
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.066762959396566e-05,
+      "loss": 1.1463,
+      "theoretical_loss": 3.4863600231379563,
+      "tokens_seen": 1818951680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.065960519980742e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.4863388589531734,
+      "tokens_seen": 1819082752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.065158080564918e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4863176967202514,
+      "tokens_seen": 1819213824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.064355641149094e-05,
+      "loss": 1.2844,
+      "theoretical_loss": 3.4862965364388705,
+      "tokens_seen": 1819344896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.06355320173327e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4862753781087097,
+      "tokens_seen": 1819475968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.062750762317445e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.4862542217294483,
+      "tokens_seen": 1819607040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.061948322901622e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.486233067300766,
+      "tokens_seen": 1819738112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.061145883485797e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.486211914822344,
+      "tokens_seen": 1819869184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.060343444069974e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.4861907642938594,
+      "tokens_seen": 1820000256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.059541004654149e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.486169615714994,
+      "tokens_seen": 1820131328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.058738565238326e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.4861484690854265,
+      "tokens_seen": 1820262400
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/docs_used": 613170,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2994256019592285,
+      "objective/train/theoretical_loss": 3.4861273244048383,
+      "objective/train/tokens_used": 190852576,
+      "theoretical_loss": 3.4861273244048383,
+      "tokens_seen": 1820393472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.057936125822501e-05,
+      "loss": 1.3112,
+      "theoretical_loss": 3.4861273244048383,
+      "tokens_seen": 1820393472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.057133686406676e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.486106181672908,
+      "tokens_seen": 1820524544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.056331246990853e-05,
+      "loss": 1.3118,
+      "theoretical_loss": 3.4860850408893165,
+      "tokens_seen": 1820655616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.055528807575028e-05,
+      "loss": 1.3171,
+      "theoretical_loss": 3.4860639020537443,
+      "tokens_seen": 1820786688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.054726368159205e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4860427651658705,
+      "tokens_seen": 1820917760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.05392392874338e-05,
+      "loss": 1.2849,
+      "theoretical_loss": 3.486021630225377,
+      "tokens_seen": 1821048832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.053121489327556e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.486000497231943,
+      "tokens_seen": 1821179904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.052319049911732e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.4859793661852496,
+      "tokens_seen": 1821310976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.051516610495908e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.4859582370849775,
+      "tokens_seen": 1821442048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.050714171080084e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4859371099308065,
+      "tokens_seen": 1821573120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.04991173166426e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.4859159847224186,
+      "tokens_seen": 1821704192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.049109292248436e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.485894861459494,
+      "tokens_seen": 1821835264
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.048306852832611e-05,
+      "loss": 1.1751,
+      "theoretical_loss": 3.485873740141713,
+      "tokens_seen": 1821966336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.047504413416787e-05,
+      "loss": 1.332,
+      "theoretical_loss": 3.4858526207687577,
+      "tokens_seen": 1822097408
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.046701974000963e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.4858315033403087,
+      "tokens_seen": 1822228480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.045899534585139e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4858103878560467,
+      "tokens_seen": 1822359552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.045097095169315e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4857892743156533,
+      "tokens_seen": 1822490624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.044294655753491e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.4857681627188097,
+      "tokens_seen": 1822621696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.043492216337666e-05,
+      "loss": 1.3175,
+      "theoretical_loss": 3.4857470530651975,
+      "tokens_seen": 1822752768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.042689776921843e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.4857259453544973,
+      "tokens_seen": 1822883840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.041887337506018e-05,
+      "loss": 1.1916,
+      "theoretical_loss": 3.485704839586391,
+      "tokens_seen": 1823014912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 9.041084898090195e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.485683735760561,
+      "tokens_seen": 1823145984
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.04028245867437e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.4856626338766876,
+      "tokens_seen": 1823277056
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.039480019258547e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.4856415339344533,
+      "tokens_seen": 1823408128
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.038677579842722e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.48562043593354,
+      "tokens_seen": 1823539200
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.037875140426897e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4855993398736294,
+      "tokens_seen": 1823670272
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.037072701011074e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.485578245754403,
+      "tokens_seen": 1823801344
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.036270261595249e-05,
+      "loss": 1.3286,
+      "theoretical_loss": 3.4855571535755434,
+      "tokens_seen": 1823932416
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.035467822179426e-05,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4855360633367325,
+      "tokens_seen": 1824063488
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.034665382763601e-05,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4855149750376526,
+      "tokens_seen": 1824194560
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.033862943347777e-05,
+      "loss": 1.3087,
+      "theoretical_loss": 3.4854938886779854,
+      "tokens_seen": 1824325632
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.033060503931953e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4854728042574137,
+      "tokens_seen": 1824456704
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.032258064516129e-05,
+      "loss": 1.3162,
+      "theoretical_loss": 3.48545172177562,
+      "tokens_seen": 1824587776
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.031455625100305e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.485430641232286,
+      "tokens_seen": 1824718848
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.03065318568448e-05,
+      "loss": 1.3085,
+      "theoretical_loss": 3.485409562627096,
+      "tokens_seen": 1824849920
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.029850746268657e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.48538848595973,
+      "tokens_seen": 1824980992
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.029048306852833e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.4853674112298734,
+      "tokens_seen": 1825112064
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.028245867437008e-05,
+      "loss": 1.3227,
+      "theoretical_loss": 3.485346338437207,
+      "tokens_seen": 1825243136
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.027443428021185e-05,
+      "loss": 1.3486,
+      "theoretical_loss": 3.485325267581414,
+      "tokens_seen": 1825374208
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.02664098860536e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.485304198662178,
+      "tokens_seen": 1825505280
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.025838549189537e-05,
+      "loss": 1.1957,
+      "theoretical_loss": 3.4852831316791817,
+      "tokens_seen": 1825636352
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.025036109773712e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.485262066632108,
+      "tokens_seen": 1825767424
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.024233670357888e-05,
+      "loss": 1.299,
+      "theoretical_loss": 3.48524100352064,
+      "tokens_seen": 1825898496
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.023431230942064e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.4852199423444614,
+      "tokens_seen": 1826029568
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.022628791526239e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.485198883103255,
+      "tokens_seen": 1826160640
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.021826352110416e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.485177825796704,
+      "tokens_seen": 1826291712
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.021023912694591e-05,
+      "loss": 1.3188,
+      "theoretical_loss": 3.4851567704244926,
+      "tokens_seen": 1826422784
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.020221473278768e-05,
+      "loss": 1.3284,
+      "theoretical_loss": 3.4851357169863038,
+      "tokens_seen": 1826553856
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.019419033862943e-05,
+      "loss": 1.2997,
+      "theoretical_loss": 3.485114665481821,
+      "tokens_seen": 1826684928
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.01861659444712e-05,
+      "loss": 1.3382,
+      "theoretical_loss": 3.485093615910728,
+      "tokens_seen": 1826816000
+    },
+    {
+      "epoch": 0.11,
+      "objective/train/docs_used": 615374,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2953439950942993,
+      "objective/train/theoretical_loss": 3.485072568272709,
+      "objective/train/tokens_used": 197406176,
+      "theoretical_loss": 3.485072568272709,
+      "tokens_seen": 1826947072
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.017814155031295e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.485072568272709,
+      "tokens_seen": 1826947072
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.01701171561547e-05,
+      "loss": 1.1792,
+      "theoretical_loss": 3.4850515225674474,
+      "tokens_seen": 1827078144
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.016209276199647e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.485030478794627,
+      "tokens_seen": 1827209216
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.015406836783822e-05,
+      "loss": 1.317,
+      "theoretical_loss": 3.4850094369539315,
+      "tokens_seen": 1827340288
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.014604397367999e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.484988397045046,
+      "tokens_seen": 1827471360
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.013801957952174e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.484967359067654,
+      "tokens_seen": 1827602432
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.012999518536351e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.484946323021439,
+      "tokens_seen": 1827733504
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.012197079120526e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.484925288906086,
+      "tokens_seen": 1827864576
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.011394639704702e-05,
+      "loss": 1.3002,
+      "theoretical_loss": 3.48490425672128,
+      "tokens_seen": 1827995648
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.010592200288878e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.484883226466704,
+      "tokens_seen": 1828126720
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.009789760873054e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.484862198142043,
+      "tokens_seen": 1828257792
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.00898732145723e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.4848411717469814,
+      "tokens_seen": 1828388864
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.008184882041406e-05,
+      "loss": 1.3434,
+      "theoretical_loss": 3.4848201472812046,
+      "tokens_seen": 1828519936
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.007382442625582e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.4847991247443963,
+      "tokens_seen": 1828651008
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.006580003209758e-05,
+      "loss": 1.2934,
+      "theoretical_loss": 3.4847781041362422,
+      "tokens_seen": 1828782080
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.005777563793933e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.484757085456427,
+      "tokens_seen": 1828913152
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.00497512437811e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.4847360687046347,
+      "tokens_seen": 1829044224
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.004172684962285e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.484715053880551,
+      "tokens_seen": 1829175296
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.003370245546462e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.484694040983861,
+      "tokens_seen": 1829306368
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.002567806130637e-05,
+      "loss": 1.3085,
+      "theoretical_loss": 3.4846730300142497,
+      "tokens_seen": 1829437440
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.001765366714814e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.484652020971402,
+      "tokens_seen": 1829568512
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.000962927298989e-05,
+      "loss": 1.3141,
+      "theoretical_loss": 3.484631013855004,
+      "tokens_seen": 1829699584
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 9.000160487883166e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.484610008664741,
+      "tokens_seen": 1829830656
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.999358048467341e-05,
+      "loss": 1.3132,
+      "theoretical_loss": 3.4845890054002973,
+      "tokens_seen": 1829961728
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.998555609051516e-05,
+      "loss": 1.3321,
+      "theoretical_loss": 3.4845680040613596,
+      "tokens_seen": 1830092800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.997753169635693e-05,
+      "loss": 1.2854,
+      "theoretical_loss": 3.484547004647613,
+      "tokens_seen": 1830223872
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.996950730219868e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.484526007158743,
+      "tokens_seen": 1830354944
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.996148290804045e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.484505011594436,
+      "tokens_seen": 1830486016
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.99534585138822e-05,
+      "loss": 1.3016,
+      "theoretical_loss": 3.484484017954377,
+      "tokens_seen": 1830617088
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.994543411972397e-05,
+      "loss": 1.3299,
+      "theoretical_loss": 3.4844630262382523,
+      "tokens_seen": 1830748160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.993740972556572e-05,
+      "loss": 1.2777,
+      "theoretical_loss": 3.4844420364457482,
+      "tokens_seen": 1830879232
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.992938533140747e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.48442104857655,
+      "tokens_seen": 1831010304
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.992136093724924e-05,
+      "loss": 1.3478,
+      "theoretical_loss": 3.4844000626303444,
+      "tokens_seen": 1831141376
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.9913336543091e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4843790786068176,
+      "tokens_seen": 1831272448
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.990531214893276e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.484358096505655,
+      "tokens_seen": 1831403520
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.989728775477451e-05,
+      "loss": 1.3238,
+      "theoretical_loss": 3.484337116326544,
+      "tokens_seen": 1831534592
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.988926336061628e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.48431613806917,
+      "tokens_seen": 1831665664
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.988123896645803e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.48429516173322,
+      "tokens_seen": 1831796736
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.987321457229979e-05,
+      "loss": 1.3161,
+      "theoretical_loss": 3.4842741873183813,
+      "tokens_seen": 1831927808
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.986519017814155e-05,
+      "loss": 1.1815,
+      "theoretical_loss": 3.4842532148243395,
+      "tokens_seen": 1832058880
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.98571657839833e-05,
+      "loss": 1.3061,
+      "theoretical_loss": 3.484232244250781,
+      "tokens_seen": 1832189952
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.984914138982507e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.4842112755973935,
+      "tokens_seen": 1832321024
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.984111699566683e-05,
+      "loss": 1.3242,
+      "theoretical_loss": 3.4841903088638637,
+      "tokens_seen": 1832452096
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.983309260150859e-05,
+      "loss": 1.31,
+      "theoretical_loss": 3.484169344049878,
+      "tokens_seen": 1832583168
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.982506820735035e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.4841483811551237,
+      "tokens_seen": 1832714240
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.98170438131921e-05,
+      "loss": 1.1641,
+      "theoretical_loss": 3.4841274201792882,
+      "tokens_seen": 1832845312
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.980901941903387e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.484106461122058,
+      "tokens_seen": 1832976384
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.980099502487562e-05,
+      "loss": 1.3036,
+      "theoretical_loss": 3.4840855039831204,
+      "tokens_seen": 1833107456
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.979297063071739e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.4840645487621633,
+      "tokens_seen": 1833238528
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.978494623655914e-05,
+      "loss": 1.334,
+      "theoretical_loss": 3.484043595458873,
+      "tokens_seen": 1833369600
+    },
+    {
+      "epoch": 0.11,
+      "objective/train/docs_used": 617542,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1996406316757202,
+      "objective/train/theoretical_loss": 3.4840226440729385,
+      "objective/train/tokens_used": 203959776,
+      "theoretical_loss": 3.4840226440729385,
+      "tokens_seen": 1833500672
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.97769218424009e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4840226440729385,
+      "tokens_seen": 1833500672
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.976889744824266e-05,
+      "loss": 1.3373,
+      "theoretical_loss": 3.4840016946040455,
+      "tokens_seen": 1833631744
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.976087305408441e-05,
+      "loss": 1.2928,
+      "theoretical_loss": 3.4839807470518833,
+      "tokens_seen": 1833762816
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.975284865992618e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.483959801416138,
+      "tokens_seen": 1833893888
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.974482426576793e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.483938857696498,
+      "tokens_seen": 1834024960
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.97367998716097e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.4839179158926514,
+      "tokens_seen": 1834156032
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.972877547745145e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.483896976004286,
+      "tokens_seen": 1834287104
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.972075108329322e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.483876038031089,
+      "tokens_seen": 1834418176
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.971272668913497e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.483855101972749,
+      "tokens_seen": 1834549248
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.970470229497674e-05,
+      "loss": 1.3085,
+      "theoretical_loss": 3.483834167828954,
+      "tokens_seen": 1834680320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.969667790081849e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.4838132355993925,
+      "tokens_seen": 1834811392
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.968865350666024e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4837923052837523,
+      "tokens_seen": 1834942464
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.968062911250201e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.483771376881722,
+      "tokens_seen": 1835073536
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.967260471834376e-05,
+      "loss": 1.3121,
+      "theoretical_loss": 3.4837504503929893,
+      "tokens_seen": 1835204608
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.966458032418553e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.4837295258172434,
+      "tokens_seen": 1835335680
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.965655593002728e-05,
+      "loss": 1.2841,
+      "theoretical_loss": 3.483708603154173,
+      "tokens_seen": 1835466752
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.964853153586905e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.483687682403466,
+      "tokens_seen": 1835597824
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.96405071417108e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.483666763564811,
+      "tokens_seen": 1835728896
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.963248274755256e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.4836458466378972,
+      "tokens_seen": 1835859968
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.962445835339432e-05,
+      "loss": 1.3584,
+      "theoretical_loss": 3.4836249316224137,
+      "tokens_seen": 1835991040
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.961643395923608e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.4836040185180486,
+      "tokens_seen": 1836122112
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.960840956507784e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.483583107324492,
+      "tokens_seen": 1836253184
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.96003851709196e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.4835621980414313,
+      "tokens_seen": 1836384256
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.959236077676136e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.4835412906685566,
+      "tokens_seen": 1836515328
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.958433638260312e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4835203852055576,
+      "tokens_seen": 1836646400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.957631198844487e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.4834994816521228,
+      "tokens_seen": 1836777472
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.956828759428664e-05,
+      "loss": 1.3312,
+      "theoretical_loss": 3.483478580007941,
+      "tokens_seen": 1836908544
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.956026320012839e-05,
+      "loss": 1.3721,
+      "theoretical_loss": 3.483457680272703,
+      "tokens_seen": 1837039616
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.955223880597016e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.4834367824460974,
+      "tokens_seen": 1837170688
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.954421441181191e-05,
+      "loss": 1.2709,
+      "theoretical_loss": 3.4834158865278138,
+      "tokens_seen": 1837301760
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.953619001765368e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.483394992517542,
+      "tokens_seen": 1837432832
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.952816562349543e-05,
+      "loss": 1.1981,
+      "theoretical_loss": 3.483374100414972,
+      "tokens_seen": 1837563904
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.952014122933718e-05,
+      "loss": 1.3137,
+      "theoretical_loss": 3.483353210219793,
+      "tokens_seen": 1837694976
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.951211683517895e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4833323219316945,
+      "tokens_seen": 1837826048
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.95040924410207e-05,
+      "loss": 1.3125,
+      "theoretical_loss": 3.483311435550367,
+      "tokens_seen": 1837957120
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.949606804686247e-05,
+      "loss": 1.2111,
+      "theoretical_loss": 3.483290551075501,
+      "tokens_seen": 1838088192
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.948804365270422e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.483269668506785,
+      "tokens_seen": 1838219264
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.948001925854599e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.4832487878439107,
+      "tokens_seen": 1838350336
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.947199486438774e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.483227909086568,
+      "tokens_seen": 1838481408
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.94639704702295e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.4832070322344464,
+      "tokens_seen": 1838612480
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.945594607607126e-05,
+      "loss": 1.2026,
+      "theoretical_loss": 3.4831861572872365,
+      "tokens_seen": 1838743552
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.944792168191301e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.4831652842446292,
+      "tokens_seen": 1838874624
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.943989728775478e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4831444131063147,
+      "tokens_seen": 1839005696
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.943187289359653e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.483123543871984,
+      "tokens_seen": 1839136768
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.94238484994383e-05,
+      "loss": 1.316,
+      "theoretical_loss": 3.4831026765413267,
+      "tokens_seen": 1839267840
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.941582410528005e-05,
+      "loss": 1.2962,
+      "theoretical_loss": 3.483081811114034,
+      "tokens_seen": 1839398912
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.94077997111218e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.483060947589797,
+      "tokens_seen": 1839529984
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 8.939977531696357e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.483040085968306,
+      "tokens_seen": 1839661056
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.939175092280533e-05,
+      "loss": 1.2837,
+      "theoretical_loss": 3.4830192262492528,
+      "tokens_seen": 1839792128
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.93837265286471e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4829983684323276,
+      "tokens_seen": 1839923200
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 619775,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.226874828338623,
+      "objective/train/theoretical_loss": 3.4829775125172215,
+      "objective/train/tokens_used": 210513376,
+      "theoretical_loss": 3.4829775125172215,
+      "tokens_seen": 1840054272
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.937570213448885e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.4829775125172215,
+      "tokens_seen": 1840054272
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.936767774033061e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.4829566585036265,
+      "tokens_seen": 1840185344
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.935965334617237e-05,
+      "loss": 1.289,
+      "theoretical_loss": 3.4829358063912323,
+      "tokens_seen": 1840316416
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.935162895201413e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.482914956179732,
+      "tokens_seen": 1840447488
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.934360455785589e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.482894107868815,
+      "tokens_seen": 1840578560
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.933558016369764e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.4828732614581743,
+      "tokens_seen": 1840709632
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.93275557695394e-05,
+      "loss": 1.3199,
+      "theoretical_loss": 3.482852416947501,
+      "tokens_seen": 1840840704
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.931953137538116e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.482831574336486,
+      "tokens_seen": 1840971776
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.931150698122293e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4828107336248215,
+      "tokens_seen": 1841102848
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.930348258706468e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.4827898948121994,
+      "tokens_seen": 1841233920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.929545819290645e-05,
+      "loss": 1.3364,
+      "theoretical_loss": 3.4827690578983113,
+      "tokens_seen": 1841364992
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.92874337987482e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4827482228828486,
+      "tokens_seen": 1841496064
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.927940940458995e-05,
+      "loss": 1.3841,
+      "theoretical_loss": 3.482727389765504,
+      "tokens_seen": 1841627136
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.927138501043172e-05,
+      "loss": 1.3449,
+      "theoretical_loss": 3.482706558545969,
+      "tokens_seen": 1841758208
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.926336061627347e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.4826857292239355,
+      "tokens_seen": 1841889280
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.925533622211524e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4826649017990965,
+      "tokens_seen": 1842020352
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.924731182795699e-05,
+      "loss": 1.3164,
+      "theoretical_loss": 3.482644076271143,
+      "tokens_seen": 1842151424
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.923928743379876e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.4826232526397685,
+      "tokens_seen": 1842282496
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.923126303964051e-05,
+      "loss": 1.1765,
+      "theoretical_loss": 3.482602430904665,
+      "tokens_seen": 1842413568
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.922323864548226e-05,
+      "loss": 1.3081,
+      "theoretical_loss": 3.482581611065524,
+      "tokens_seen": 1842544640
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.921521425132403e-05,
+      "loss": 1.3231,
+      "theoretical_loss": 3.4825607931220395,
+      "tokens_seen": 1842675712
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.920718985716578e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4825399770739027,
+      "tokens_seen": 1842806784
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.919916546300755e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.4825191629208074,
+      "tokens_seen": 1842937856
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.91911410688493e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4824983506624454,
+      "tokens_seen": 1843068928
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.918311667469107e-05,
+      "loss": 1.1829,
+      "theoretical_loss": 3.48247754029851,
+      "tokens_seen": 1843200000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.917509228053282e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4824567318286945,
+      "tokens_seen": 1843331072
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.916706788637458e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.482435925252691,
+      "tokens_seen": 1843462144
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.915904349221634e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4824151205701925,
+      "tokens_seen": 1843593216
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.91510190980581e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.4823943177808925,
+      "tokens_seen": 1843724288
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.914299470389986e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.4823735168844845,
+      "tokens_seen": 1843855360
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.913497030974162e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.482352717880661,
+      "tokens_seen": 1843986432
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.912694591558338e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.4823319207691155,
+      "tokens_seen": 1844117504
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.911892152142514e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.482311125549541,
+      "tokens_seen": 1844248576
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.911089712726689e-05,
+      "loss": 1.1684,
+      "theoretical_loss": 3.482290332221632,
+      "tokens_seen": 1844379648
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.910287273310866e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.482269540785081,
+      "tokens_seen": 1844510720
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.909484833895041e-05,
+      "loss": 1.2723,
+      "theoretical_loss": 3.482248751239582,
+      "tokens_seen": 1844641792
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.908682394479218e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.482227963584829,
+      "tokens_seen": 1844772864
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.907879955063393e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.4822071778205146,
+      "tokens_seen": 1844903936
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.90707751564757e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.482186393946334,
+      "tokens_seen": 1845035008
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.906275076231745e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.48216561196198,
+      "tokens_seen": 1845166080
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.905472636815922e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.482144831867147,
+      "tokens_seen": 1845297152
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.904670197400097e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.482124053661529,
+      "tokens_seen": 1845428224
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.903867757984272e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.48210327734482,
+      "tokens_seen": 1845559296
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.903065318568449e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.482082502916714,
+      "tokens_seen": 1845690368
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.902262879152624e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4820617303769055,
+      "tokens_seen": 1845821440
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.901460439736801e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.4820409597250888,
+      "tokens_seen": 1845952512
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.900658000320976e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.482020190960958,
+      "tokens_seen": 1846083584
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.899855560905153e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.481999424084208,
+      "tokens_seen": 1846214656
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.899053121489328e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.481978659094532,
+      "tokens_seen": 1846345728
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.898250682073503e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.481957895991626,
+      "tokens_seen": 1846476800
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 621685,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3204975128173828,
+      "objective/train/theoretical_loss": 3.4819371347751846,
+      "objective/train/tokens_used": 217066976,
+      "theoretical_loss": 3.4819371347751846,
+      "tokens_seen": 1846607872
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.89744824265768e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4819371347751846,
+      "tokens_seen": 1846607872
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.896645803241855e-05,
+      "loss": 1.2917,
+      "theoretical_loss": 3.481916375444902,
+      "tokens_seen": 1846738944
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.895843363826032e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.4818956180004728,
+      "tokens_seen": 1846870016
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.895040924410207e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4818748624415923,
+      "tokens_seen": 1847001088
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.894238484994384e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4818541087679558,
+      "tokens_seen": 1847132160
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.89343604557856e-05,
+      "loss": 1.3677,
+      "theoretical_loss": 3.4818333569792568,
+      "tokens_seen": 1847263232
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.892633606162735e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.4818126070751925,
+      "tokens_seen": 1847394304
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.891831166746911e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.4817918590554564,
+      "tokens_seen": 1847525376
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.891028727331087e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.481771112919744,
+      "tokens_seen": 1847656448
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.890226287915263e-05,
+      "loss": 1.362,
+      "theoretical_loss": 3.481750368667751,
+      "tokens_seen": 1847787520
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.889423848499439e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.4817296262991726,
+      "tokens_seen": 1847918592
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.888621409083615e-05,
+      "loss": 1.3194,
+      "theoretical_loss": 3.4817088858137044,
+      "tokens_seen": 1848049664
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.88781896966779e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.4816881472110417,
+      "tokens_seen": 1848180736
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.887016530251966e-05,
+      "loss": 1.322,
+      "theoretical_loss": 3.48166741049088,
+      "tokens_seen": 1848311808
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.886214090836143e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4816466756529145,
+      "tokens_seen": 1848442880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.885411651420318e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.481625942696842,
+      "tokens_seen": 1848573952
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.884609212004495e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.481605211622358,
+      "tokens_seen": 1848705024
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.88380677258867e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4815844824291577,
+      "tokens_seen": 1848836096
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.883004333172847e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.4815637551169374,
+      "tokens_seen": 1848967168
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.882201893757022e-05,
+      "loss": 1.3493,
+      "theoretical_loss": 3.481543029685393,
+      "tokens_seen": 1849098240
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.881399454341197e-05,
+      "loss": 1.3391,
+      "theoretical_loss": 3.481522306134221,
+      "tokens_seen": 1849229312
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.880597014925374e-05,
+      "loss": 1.17,
+      "theoretical_loss": 3.4815015844631167,
+      "tokens_seen": 1849360384
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.879794575509549e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.481480864671777,
+      "tokens_seen": 1849491456
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.878992136093726e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.4814601467598987,
+      "tokens_seen": 1849622528
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.878189696677901e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.481439430727177,
+      "tokens_seen": 1849753600
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.877387257262078e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.4814187165733084,
+      "tokens_seen": 1849884672
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.876584817846253e-05,
+      "loss": 1.3263,
+      "theoretical_loss": 3.48139800429799,
+      "tokens_seen": 1850015744
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.87578237843043e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.481377293900918,
+      "tokens_seen": 1850146816
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.874979939014605e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.481356585381789,
+      "tokens_seen": 1850277888
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.87417749959878e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.4813358787403006,
+      "tokens_seen": 1850408960
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.873375060182957e-05,
+      "loss": 1.2656,
+      "theoretical_loss": 3.481315173976148,
+      "tokens_seen": 1850540032
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.872572620767132e-05,
+      "loss": 1.3465,
+      "theoretical_loss": 3.4812944710890292,
+      "tokens_seen": 1850671104
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.871770181351309e-05,
+      "loss": 1.2799,
+      "theoretical_loss": 3.4812737700786407,
+      "tokens_seen": 1850802176
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.870967741935484e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.4812530709446796,
+      "tokens_seen": 1850933248
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.870165302519661e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.481232373686843,
+      "tokens_seen": 1851064320
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.869362863103836e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.4812116783048275,
+      "tokens_seen": 1851195392
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.868560423688012e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.4811909847983316,
+      "tokens_seen": 1851326464
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.867757984272188e-05,
+      "loss": 1.1545,
+      "theoretical_loss": 3.4811702931670507,
+      "tokens_seen": 1851457536
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.866955544856364e-05,
+      "loss": 1.185,
+      "theoretical_loss": 3.4811496034106835,
+      "tokens_seen": 1851588608
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.86615310544054e-05,
+      "loss": 1.2643,
+      "theoretical_loss": 3.4811289155289273,
+      "tokens_seen": 1851719680
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.865350666024716e-05,
+      "loss": 1.3373,
+      "theoretical_loss": 3.481108229521479,
+      "tokens_seen": 1851850752
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.864548226608892e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4810875453880366,
+      "tokens_seen": 1851981824
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.863745787193068e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.481066863128297,
+      "tokens_seen": 1852112896
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.862943347777243e-05,
+      "loss": 1.3233,
+      "theoretical_loss": 3.4810461827419594,
+      "tokens_seen": 1852243968
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.86214090836142e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.48102550422872,
+      "tokens_seen": 1852375040
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.861338468945595e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.4810048275882775,
+      "tokens_seen": 1852506112
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.860536029529772e-05,
+      "loss": 1.1729,
+      "theoretical_loss": 3.480984152820329,
+      "tokens_seen": 1852637184
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.859733590113947e-05,
+      "loss": 1.3373,
+      "theoretical_loss": 3.4809634799245734,
+      "tokens_seen": 1852768256
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.858931150698124e-05,
+      "loss": 1.3154,
+      "theoretical_loss": 3.4809428089007084,
+      "tokens_seen": 1852899328
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.858128711282299e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.480922139748432,
+      "tokens_seen": 1853030400
+    },
+    {
+      "epoch": 0.12,
+      "objective/train/docs_used": 624097,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.226438045501709,
+      "objective/train/theoretical_loss": 3.4809014724674423,
+      "objective/train/tokens_used": 223620576,
+      "theoretical_loss": 3.4809014724674423,
+      "tokens_seen": 1853161472
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.857326271866474e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.4809014724674423,
+      "tokens_seen": 1853161472
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.856523832450651e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.4808808070574377,
+      "tokens_seen": 1853292544
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.855721393034826e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.4808601435181163,
+      "tokens_seen": 1853423616
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.854918953619003e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.480839481849177,
+      "tokens_seen": 1853554688
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.854116514203178e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.480818822050318,
+      "tokens_seen": 1853685760
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.853314074787355e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4807981641212375,
+      "tokens_seen": 1853816832
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.85251163537153e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.4807775080616343,
+      "tokens_seen": 1853947904
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.851709195955705e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.480756853871207,
+      "tokens_seen": 1854078976
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.850906756539882e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.4807362015496546,
+      "tokens_seen": 1854210048
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.850104317124057e-05,
+      "loss": 1.2838,
+      "theoretical_loss": 3.480715551096676,
+      "tokens_seen": 1854341120
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.849301877708234e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.4806949025119693,
+      "tokens_seen": 1854472192
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.84849943829241e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.4806742557952344,
+      "tokens_seen": 1854603264
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.847696998876586e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4806536109461694,
+      "tokens_seen": 1854734336
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.846894559460761e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.4806329679644743,
+      "tokens_seen": 1854865408
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.846092120044937e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4806123268498474,
+      "tokens_seen": 1854996480
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.845289680629113e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.480591687601989,
+      "tokens_seen": 1855127552
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.844487241213289e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4805710502205973,
+      "tokens_seen": 1855258624
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.843684801797465e-05,
+      "loss": 1.2169,
+      "theoretical_loss": 3.4805504147053723,
+      "tokens_seen": 1855389696
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.842882362381641e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.4805297810560134,
+      "tokens_seen": 1855520768
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.842079922965817e-05,
+      "loss": 1.2802,
+      "theoretical_loss": 3.4805091492722195,
+      "tokens_seen": 1855651840
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.841277483549993e-05,
+      "loss": 1.1763,
+      "theoretical_loss": 3.480488519353691,
+      "tokens_seen": 1855782912
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.84047504413417e-05,
+      "loss": 1.2878,
+      "theoretical_loss": 3.480467891300127,
+      "tokens_seen": 1855913984
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.839672604718345e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.480447265111227,
+      "tokens_seen": 1856045056
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 8.83887016530252e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.4804266407866917,
+      "tokens_seen": 1856176128
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.838067725886697e-05,
+      "loss": 1.1616,
+      "theoretical_loss": 3.48040601832622,
+      "tokens_seen": 1856307200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.837265286470872e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.4803853977295125,
+      "tokens_seen": 1856438272
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.836462847055049e-05,
+      "loss": 1.1551,
+      "theoretical_loss": 3.480364778996269,
+      "tokens_seen": 1856569344
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.835660407639224e-05,
+      "loss": 1.3051,
+      "theoretical_loss": 3.4803441621261895,
+      "tokens_seen": 1856700416
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.8348579682234e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.4803235471189735,
+      "tokens_seen": 1856831488
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.834055528807576e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.4803029339743228,
+      "tokens_seen": 1856962560
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.833253089391751e-05,
+      "loss": 1.3312,
+      "theoretical_loss": 3.480282322691936,
+      "tokens_seen": 1857093632
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.832450649975928e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.4802617132715143,
+      "tokens_seen": 1857224704
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.831648210560103e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4802411057127576,
+      "tokens_seen": 1857355776
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.83084577114428e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.480220500015367,
+      "tokens_seen": 1857486848
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.830043331728455e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.480199896179043,
+      "tokens_seen": 1857617920
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.829240892312632e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.4801792942034853,
+      "tokens_seen": 1857748992
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.828438452896807e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.480158694088396,
+      "tokens_seen": 1857880064
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.827636013480982e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.480138095833474,
+      "tokens_seen": 1858011136
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.826833574065159e-05,
+      "loss": 1.2844,
+      "theoretical_loss": 3.480117499438422,
+      "tokens_seen": 1858142208
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.826031134649334e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.48009690490294,
+      "tokens_seen": 1858273280
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.825228695233511e-05,
+      "loss": 1.2602,
+      "theoretical_loss": 3.4800763122267293,
+      "tokens_seen": 1858404352
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.824426255817686e-05,
+      "loss": 1.3044,
+      "theoretical_loss": 3.4800557214094905,
+      "tokens_seen": 1858535424
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.823623816401863e-05,
+      "loss": 1.3232,
+      "theoretical_loss": 3.480035132450925,
+      "tokens_seen": 1858666496
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.822821376986038e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.480014545350733,
+      "tokens_seen": 1858797568
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.822018937570214e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.479993960108618,
+      "tokens_seen": 1858928640
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.82121649815439e-05,
+      "loss": 1.2902,
+      "theoretical_loss": 3.4799733767242786,
+      "tokens_seen": 1859059712
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.820414058738566e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.479952795197418,
+      "tokens_seen": 1859190784
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.819611619322742e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.4799322155277377,
+      "tokens_seen": 1859321856
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.818809179906918e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.479911637714938,
+      "tokens_seen": 1859452928
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.818006740491094e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.479891061758721,
+      "tokens_seen": 1859584000
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 626401,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2142536640167236,
+      "objective/train/theoretical_loss": 3.479870487658789,
+      "objective/train/tokens_used": 230174176,
+      "theoretical_loss": 3.479870487658789,
+      "tokens_seen": 1859715072
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.81720430107527e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.479870487658789,
+      "tokens_seen": 1859715072
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.816401861659445e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.479849915414843,
+      "tokens_seen": 1859846144
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.815599422243622e-05,
+      "loss": 1.326,
+      "theoretical_loss": 3.4798293450265847,
+      "tokens_seen": 1859977216
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.814796982827797e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.4798087764937167,
+      "tokens_seen": 1860108288
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.813994543411974e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.47978820981594,
+      "tokens_seen": 1860239360
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.813192103996149e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.479767644992957,
+      "tokens_seen": 1860370432
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.812389664580326e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.4797470820244705,
+      "tokens_seen": 1860501504
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.811587225164501e-05,
+      "loss": 1.3197,
+      "theoretical_loss": 3.479726520910181,
+      "tokens_seen": 1860632576
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.810784785748678e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.4797059616497927,
+      "tokens_seen": 1860763648
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.809982346332853e-05,
+      "loss": 1.3826,
+      "theoretical_loss": 3.479685404243006,
+      "tokens_seen": 1860894720
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.809179906917028e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.4796648486895245,
+      "tokens_seen": 1861025792
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.808377467501205e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4796442949890505,
+      "tokens_seen": 1861156864
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.80757502808538e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.4796237431412855,
+      "tokens_seen": 1861287936
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.806772588669557e-05,
+      "loss": 1.3028,
+      "theoretical_loss": 3.479603193145933,
+      "tokens_seen": 1861419008
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.805970149253732e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4795826450026954,
+      "tokens_seen": 1861550080
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.805167709837908e-05,
+      "loss": 1.2723,
+      "theoretical_loss": 3.479562098711275,
+      "tokens_seen": 1861681152
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.804365270422084e-05,
+      "loss": 1.3792,
+      "theoretical_loss": 3.4795415542713757,
+      "tokens_seen": 1861812224
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.80356283100626e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.4795210116826985,
+      "tokens_seen": 1861943296
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.802760391590436e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.4795004709449477,
+      "tokens_seen": 1862074368
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.801957952174611e-05,
+      "loss": 1.3058,
+      "theoretical_loss": 3.479479932057826,
+      "tokens_seen": 1862205440
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.801155512758788e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4794593950210357,
+      "tokens_seen": 1862336512
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.800353073342963e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.479438859834281,
+      "tokens_seen": 1862467584
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.799550633927139e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4794183264972647,
+      "tokens_seen": 1862598656
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.798748194511315e-05,
+      "loss": 1.2184,
+      "theoretical_loss": 3.4793977950096893,
+      "tokens_seen": 1862729728
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.797945755095491e-05,
+      "loss": 1.3183,
+      "theoretical_loss": 3.4793772653712587,
+      "tokens_seen": 1862860800
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.797143315679667e-05,
+      "loss": 1.3463,
+      "theoretical_loss": 3.4793567375816767,
+      "tokens_seen": 1862991872
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.796340876263843e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.4793362116406454,
+      "tokens_seen": 1863122944
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.795538436848018e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.47931568754787,
+      "tokens_seen": 1863254016
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.794735997432195e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.4792951653030526,
+      "tokens_seen": 1863385088
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.79393355801637e-05,
+      "loss": 1.2127,
+      "theoretical_loss": 3.479274644905898,
+      "tokens_seen": 1863516160
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.793131118600547e-05,
+      "loss": 1.3125,
+      "theoretical_loss": 3.4792541263561096,
+      "tokens_seen": 1863647232
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.792328679184722e-05,
+      "loss": 1.3227,
+      "theoretical_loss": 3.4792336096533907,
+      "tokens_seen": 1863778304
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.791526239768899e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.479213094797445,
+      "tokens_seen": 1863909376
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.790723800353074e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.479192581787977,
+      "tokens_seen": 1864040448
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.789921360937249e-05,
+      "loss": 1.2861,
+      "theoretical_loss": 3.479172070624691,
+      "tokens_seen": 1864171520
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.789118921521426e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.4791515613072903,
+      "tokens_seen": 1864302592
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.788316482105601e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.479131053835479,
+      "tokens_seen": 1864433664
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.787514042689778e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.479110548208962,
+      "tokens_seen": 1864564736
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.786711603273953e-05,
+      "loss": 1.3226,
+      "theoretical_loss": 3.479090044427443,
+      "tokens_seen": 1864695808
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.785909163858129e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4790695424906266,
+      "tokens_seen": 1864826880
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.785106724442305e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.479049042398217,
+      "tokens_seen": 1864957952
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.78430428502648e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.4790285441499185,
+      "tokens_seen": 1865089024
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.783501845610657e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.479008047745436,
+      "tokens_seen": 1865220096
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.782699406194833e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.4789875531844743,
+      "tokens_seen": 1865351168
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.781896966779009e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4789670604667373,
+      "tokens_seen": 1865482240
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.781094527363185e-05,
+      "loss": 1.3032,
+      "theoretical_loss": 3.47894656959193,
+      "tokens_seen": 1865613312
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.78029208794736e-05,
+      "loss": 1.3786,
+      "theoretical_loss": 3.478926080559758,
+      "tokens_seen": 1865744384
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.779489648531536e-05,
+      "loss": 1.3176,
+      "theoretical_loss": 3.478905593369925,
+      "tokens_seen": 1865875456
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.778687209115712e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.478885108022137,
+      "tokens_seen": 1866006528
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.777884769699888e-05,
+      "loss": 1.3015,
+      "theoretical_loss": 3.478864624516098,
+      "tokens_seen": 1866137600
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 628509,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2967095375061035,
+      "objective/train/theoretical_loss": 3.4788441428515133,
+      "objective/train/tokens_used": 236727776,
+      "theoretical_loss": 3.4788441428515133,
+      "tokens_seen": 1866268672
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.777082330284064e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4788441428515133,
+      "tokens_seen": 1866268672
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.776279890868239e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4788236630280887,
+      "tokens_seen": 1866399744
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.775477451452416e-05,
+      "loss": 1.3076,
+      "theoretical_loss": 3.478803185045529,
+      "tokens_seen": 1866530816
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.774675012036591e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4787827089035397,
+      "tokens_seen": 1866661888
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.773872572620768e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4787622346018257,
+      "tokens_seen": 1866792960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.773070133204943e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.4787417621400927,
+      "tokens_seen": 1866924032
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.77226769378912e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.4787212915180463,
+      "tokens_seen": 1867055104
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.771465254373295e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.4787008227353917,
+      "tokens_seen": 1867186176
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.77066281495747e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4786803557918344,
+      "tokens_seen": 1867317248
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.769860375541647e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.478659890687081,
+      "tokens_seen": 1867448320
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.769057936125822e-05,
+      "loss": 1.2822,
+      "theoretical_loss": 3.478639427420836,
+      "tokens_seen": 1867579392
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.768255496709999e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.4786189659928057,
+      "tokens_seen": 1867710464
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.767453057294174e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.478598506402697,
+      "tokens_seen": 1867841536
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.76665061787835e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.4785780486502142,
+      "tokens_seen": 1867972608
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.765848178462526e-05,
+      "loss": 1.3356,
+      "theoretical_loss": 3.478557592735065,
+      "tokens_seen": 1868103680
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.765045739046702e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.478537138656954,
+      "tokens_seen": 1868234752
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.764243299630878e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4785166864155883,
+      "tokens_seen": 1868365824
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.763440860215054e-05,
+      "loss": 1.337,
+      "theoretical_loss": 3.478496236010673,
+      "tokens_seen": 1868496896
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.76263842079923e-05,
+      "loss": 1.3656,
+      "theoretical_loss": 3.478475787441916,
+      "tokens_seen": 1868627968
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.761835981383406e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.478455340709023,
+      "tokens_seen": 1868759040
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.761033541967581e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.4784348958116995,
+      "tokens_seen": 1868890112
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.760231102551758e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4784144527496537,
+      "tokens_seen": 1869021184
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.759428663135933e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4783940115225906,
+      "tokens_seen": 1869152256
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.75862622372011e-05,
+      "loss": 1.3382,
+      "theoretical_loss": 3.4783735721302174,
+      "tokens_seen": 1869283328
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.757823784304285e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.478353134572241,
+      "tokens_seen": 1869414400
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.75702134488846e-05,
+      "loss": 1.1642,
+      "theoretical_loss": 3.4783326988483685,
+      "tokens_seen": 1869545472
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.756218905472637e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.478312264958306,
+      "tokens_seen": 1869676544
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.755416466056812e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.4782918329017605,
+      "tokens_seen": 1869807616
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.754614026640989e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4782714026784394,
+      "tokens_seen": 1869938688
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.753811587225164e-05,
+      "loss": 1.3297,
+      "theoretical_loss": 3.4782509742880494,
+      "tokens_seen": 1870069760
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.753009147809341e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.4782305477302975,
+      "tokens_seen": 1870200832
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.752206708393516e-05,
+      "loss": 1.3332,
+      "theoretical_loss": 3.4782101230048914,
+      "tokens_seen": 1870331904
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.751404268977691e-05,
+      "loss": 1.2861,
+      "theoretical_loss": 3.4781897001115376,
+      "tokens_seen": 1870462976
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.750601829561868e-05,
+      "loss": 1.3045,
+      "theoretical_loss": 3.4781692790499443,
+      "tokens_seen": 1870594048
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.749799390146043e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.478148859819818,
+      "tokens_seen": 1870725120
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.74899695073022e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4781284424208665,
+      "tokens_seen": 1870856192
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.748194511314395e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.478108026852797,
+      "tokens_seen": 1870987264
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.747392071898572e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4780876131153184,
+      "tokens_seen": 1871118336
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.746589632482747e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4780672012081366,
+      "tokens_seen": 1871249408
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.745787193066924e-05,
+      "loss": 1.3138,
+      "theoretical_loss": 3.4780467911309603,
+      "tokens_seen": 1871380480
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.7449847536511e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.478026382883497,
+      "tokens_seen": 1871511552
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.744182314235275e-05,
+      "loss": 1.3112,
+      "theoretical_loss": 3.4780059764654547,
+      "tokens_seen": 1871642624
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.743379874819451e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.477985571876541,
+      "tokens_seen": 1871773696
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.742577435403627e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.477965169116464,
+      "tokens_seen": 1871904768
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.741774995987803e-05,
+      "loss": 1.2816,
+      "theoretical_loss": 3.477944768184932,
+      "tokens_seen": 1872035840
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.740972556571979e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.477924369081653,
+      "tokens_seen": 1872166912
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.740170117156155e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.4779039718063345,
+      "tokens_seen": 1872297984
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.73936767774033e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.477883576358686,
+      "tokens_seen": 1872429056
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.738565238324506e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4778631827384148,
+      "tokens_seen": 1872560128
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 8.737762798908683e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.47784279094523,
+      "tokens_seen": 1872691200
+    },
+    {
+      "epoch": 0.13,
+      "objective/train/docs_used": 630672,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2571489810943604,
+      "objective/train/theoretical_loss": 3.477822400978839,
+      "objective/train/tokens_used": 243281376,
+      "theoretical_loss": 3.477822400978839,
+      "tokens_seen": 1872822272
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.736960359492858e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.477822400978839,
+      "tokens_seen": 1872822272
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.736157920077035e-05,
+      "loss": 1.2925,
+      "theoretical_loss": 3.4778020128389517,
+      "tokens_seen": 1872953344
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.73535548066121e-05,
+      "loss": 1.2968,
+      "theoretical_loss": 3.477781626525276,
+      "tokens_seen": 1873084416
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.734553041245387e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.47776124203752,
+      "tokens_seen": 1873215488
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.733750601829562e-05,
+      "loss": 1.3481,
+      "theoretical_loss": 3.477740859375393,
+      "tokens_seen": 1873346560
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.732948162413737e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.477720478538604,
+      "tokens_seen": 1873477632
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.732145722997914e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.4777000995268614,
+      "tokens_seen": 1873608704
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.731343283582089e-05,
+      "loss": 1.3474,
+      "theoretical_loss": 3.4776797223398743,
+      "tokens_seen": 1873739776
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.730540844166266e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.477659346977352,
+      "tokens_seen": 1873870848
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.729738404750441e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.477638973439003,
+      "tokens_seen": 1874001920
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.728935965334618e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4776186017245365,
+      "tokens_seen": 1874132992
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.728133525918793e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.4775982318336616,
+      "tokens_seen": 1874264064
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.727331086502968e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.477577863766088,
+      "tokens_seen": 1874395136
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.726528647087145e-05,
+      "loss": 1.3442,
+      "theoretical_loss": 3.477557497521525,
+      "tokens_seen": 1874526208
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.72572620767132e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.4775371330996814,
+      "tokens_seen": 1874657280
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.724923768255497e-05,
+      "loss": 1.3216,
+      "theoretical_loss": 3.4775167705002668,
+      "tokens_seen": 1874788352
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.724121328839672e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.4774964097229915,
+      "tokens_seen": 1874919424
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.723318889423849e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.477476050767564,
+      "tokens_seen": 1875050496
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.722516450008024e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.477455693633695,
+      "tokens_seen": 1875181568
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.7217140105922e-05,
+      "loss": 1.3765,
+      "theoretical_loss": 3.477435338321093,
+      "tokens_seen": 1875312640
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.720911571176376e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.477414984829469,
+      "tokens_seen": 1875443712
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.720109131760552e-05,
+      "loss": 1.3023,
+      "theoretical_loss": 3.477394633158532,
+      "tokens_seen": 1875574784
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.719306692344728e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.4773742833079924,
+      "tokens_seen": 1875705856
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.718504252928904e-05,
+      "loss": 1.311,
+      "theoretical_loss": 3.4773539352775593,
+      "tokens_seen": 1875836928
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.71770181351308e-05,
+      "loss": 1.3824,
+      "theoretical_loss": 3.4773335890669443,
+      "tokens_seen": 1875968000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.716899374097256e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.477313244675856,
+      "tokens_seen": 1876099072
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.716096934681432e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.477292902104006,
+      "tokens_seen": 1876230144
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.715294495265608e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.477272561351103,
+      "tokens_seen": 1876361216
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.714492055849783e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.4772522224168583,
+      "tokens_seen": 1876492288
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.71368961643396e-05,
+      "loss": 1.2628,
+      "theoretical_loss": 3.4772318853009825,
+      "tokens_seen": 1876623360
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.712887177018135e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.477211550003185,
+      "tokens_seen": 1876754432
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.712084737602312e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.4771912165231775,
+      "tokens_seen": 1876885504
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.711282298186487e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.4771708848606693,
+      "tokens_seen": 1877016576
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.710479858770664e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.4771505550153723,
+      "tokens_seen": 1877147648
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.709677419354839e-05,
+      "loss": 1.2833,
+      "theoretical_loss": 3.477130226986996,
+      "tokens_seen": 1877278720
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.708874979939014e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.4771099007752526,
+      "tokens_seen": 1877409792
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.708072540523191e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.4770895763798517,
+      "tokens_seen": 1877540864
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.707270101107366e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4770692538005052,
+      "tokens_seen": 1877671936
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.706467661691543e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.477048933036923,
+      "tokens_seen": 1877803008
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.705665222275718e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.477028614088817,
+      "tokens_seen": 1877934080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.704862782859895e-05,
+      "loss": 1.3389,
+      "theoretical_loss": 3.4770082969558977,
+      "tokens_seen": 1878065152
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.70406034344407e-05,
+      "loss": 1.17,
+      "theoretical_loss": 3.476987981637877,
+      "tokens_seen": 1878196224
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.703257904028245e-05,
+      "loss": 1.3049,
+      "theoretical_loss": 3.4769676681344652,
+      "tokens_seen": 1878327296
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.702455464612422e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.476947356445375,
+      "tokens_seen": 1878458368
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.701653025196597e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4769270465703164,
+      "tokens_seen": 1878589440
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.700850585780774e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.476906738509001,
+      "tokens_seen": 1878720512
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.70004814636495e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.476886432261141,
+      "tokens_seen": 1878851584
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.699245706949126e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.4768661278264474,
+      "tokens_seen": 1878982656
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.698443267533301e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.4768458252046326,
+      "tokens_seen": 1879113728
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.697640828117477e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.476825524395407,
+      "tokens_seen": 1879244800
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 632825,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2842808961868286,
+      "objective/train/theoretical_loss": 3.4768052253984836,
+      "objective/train/tokens_used": 249834976,
+      "theoretical_loss": 3.4768052253984836,
+      "tokens_seen": 1879375872
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.696838388701653e-05,
+      "loss": 1.3235,
+      "theoretical_loss": 3.4768052253984836,
+      "tokens_seen": 1879375872
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.696035949285829e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.476784928213574,
+      "tokens_seen": 1879506944
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.695233509870005e-05,
+      "loss": 1.3041,
+      "theoretical_loss": 3.476764632840389,
+      "tokens_seen": 1879638016
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.69443107045418e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.4767443392786426,
+      "tokens_seen": 1879769088
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.693628631038357e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4767240475280445,
+      "tokens_seen": 1879900160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.692826191622533e-05,
+      "loss": 1.287,
+      "theoretical_loss": 3.476703757588308,
+      "tokens_seen": 1880031232
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.692023752206708e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.476683469459146,
+      "tokens_seen": 1880162304
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.691221312790885e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.47666318314027,
+      "tokens_seen": 1880293376
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.69041887337506e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4766428986313915,
+      "tokens_seen": 1880424448
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.689616433959237e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.476622615932224,
+      "tokens_seen": 1880555520
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.688813994543412e-05,
+      "loss": 1.2975,
+      "theoretical_loss": 3.4766023350424797,
+      "tokens_seen": 1880686592
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.688011555127589e-05,
+      "loss": 1.3387,
+      "theoretical_loss": 3.4765820559618703,
+      "tokens_seen": 1880817664
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.687209115711764e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.47656177869011,
+      "tokens_seen": 1880948736
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.68640667629594e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.476541503226909,
+      "tokens_seen": 1881079808
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.685604236880116e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.4765212295719827,
+      "tokens_seen": 1881210880
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.684801797464291e-05,
+      "loss": 1.304,
+      "theoretical_loss": 3.476500957725042,
+      "tokens_seen": 1881341952
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.683999358048468e-05,
+      "loss": 1.3495,
+      "theoretical_loss": 3.4764806876858003,
+      "tokens_seen": 1881473024
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.683196918632643e-05,
+      "loss": 1.3245,
+      "theoretical_loss": 3.4764604194539706,
+      "tokens_seen": 1881604096
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.68239447921682e-05,
+      "loss": 1.3773,
+      "theoretical_loss": 3.4764401530292655,
+      "tokens_seen": 1881735168
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.681592039800995e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.476419888411398,
+      "tokens_seen": 1881866240
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.680789600385172e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.4763996256000818,
+      "tokens_seen": 1881997312
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.679987160969347e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.47637936459503,
+      "tokens_seen": 1882128384
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.679184721553522e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4763591053959546,
+      "tokens_seen": 1882259456
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.678382282137699e-05,
+      "loss": 1.2933,
+      "theoretical_loss": 3.4763388480025705,
+      "tokens_seen": 1882390528
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.677579842721874e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.47631859241459,
+      "tokens_seen": 1882521600
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.676777403306051e-05,
+      "loss": 1.3375,
+      "theoretical_loss": 3.4762983386317265,
+      "tokens_seen": 1882652672
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.675974963890226e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.476278086653694,
+      "tokens_seen": 1882783744
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.675172524474403e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.4762578364802064,
+      "tokens_seen": 1882914816
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.674370085058578e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.476237588110976,
+      "tokens_seen": 1883045888
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.673567645642754e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.4762173415457176,
+      "tokens_seen": 1883176960
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.67276520622693e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4761970967841442,
+      "tokens_seen": 1883308032
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.671962766811106e-05,
+      "loss": 1.3069,
+      "theoretical_loss": 3.47617685382597,
+      "tokens_seen": 1883439104
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.671160327395282e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.476156612670909,
+      "tokens_seen": 1883570176
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.670357887979458e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.476136373318675,
+      "tokens_seen": 1883701248
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.669555448563634e-05,
+      "loss": 1.3454,
+      "theoretical_loss": 3.476116135768982,
+      "tokens_seen": 1883832320
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.66875300914781e-05,
+      "loss": 1.3017,
+      "theoretical_loss": 3.476095900021544,
+      "tokens_seen": 1883963392
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.667950569731985e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.476075666076075,
+      "tokens_seen": 1884094464
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.667148130316162e-05,
+      "loss": 1.3105,
+      "theoretical_loss": 3.4760554339322893,
+      "tokens_seen": 1884225536
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.666345690900337e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.476035203589901,
+      "tokens_seen": 1884356608
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.665543251484514e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.476014975048625,
+      "tokens_seen": 1884487680
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.664740812068689e-05,
+      "loss": 1.3008,
+      "theoretical_loss": 3.4759947483081755,
+      "tokens_seen": 1884618752
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.663938372652866e-05,
+      "loss": 1.3342,
+      "theoretical_loss": 3.475974523368266,
+      "tokens_seen": 1884749824
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.663135933237041e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.4759543002286124,
+      "tokens_seen": 1884880896
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.662333493821216e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.475934078888929,
+      "tokens_seen": 1885011968
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.661531054405393e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.475913859348929,
+      "tokens_seen": 1885143040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.660728614989568e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4758936416083293,
+      "tokens_seen": 1885274112
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.659926175573745e-05,
+      "loss": 1.1978,
+      "theoretical_loss": 3.475873425666843,
+      "tokens_seen": 1885405184
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.65912373615792e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4758532115241856,
+      "tokens_seen": 1885536256
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.658321296742097e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.4758329991800725,
+      "tokens_seen": 1885667328
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.657518857326272e-05,
+      "loss": 1.2934,
+      "theoretical_loss": 3.475812788634218,
+      "tokens_seen": 1885798400
+    },
+    {
+      "epoch": 0.14,
+      "objective/train/docs_used": 635135,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.216483235359192,
+      "objective/train/theoretical_loss": 3.475792579886337,
+      "objective/train/tokens_used": 256388576,
+      "theoretical_loss": 3.475792579886337,
+      "tokens_seen": 1885929472
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.656716417910447e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.475792579886337,
+      "tokens_seen": 1885929472
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.655913978494624e-05,
+      "loss": 1.3344,
+      "theoretical_loss": 3.475772372936145,
+      "tokens_seen": 1886060544
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.6551115390788e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.475752167783357,
+      "tokens_seen": 1886191616
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.654309099662976e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4757319644276885,
+      "tokens_seen": 1886322688
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.653506660247151e-05,
+      "loss": 1.3124,
+      "theoretical_loss": 3.4757117628688543,
+      "tokens_seen": 1886453760
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.652704220831328e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.4756915631065706,
+      "tokens_seen": 1886584832
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.651901781415503e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.4756713651405517,
+      "tokens_seen": 1886715904
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.65109934199968e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.475651168970514,
+      "tokens_seen": 1886846976
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.650296902583855e-05,
+      "loss": 1.3055,
+      "theoretical_loss": 3.475630974596173,
+      "tokens_seen": 1886978048
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.649494463168031e-05,
+      "loss": 1.3315,
+      "theoretical_loss": 3.4756107820172435,
+      "tokens_seen": 1887109120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.648692023752207e-05,
+      "loss": 1.3207,
+      "theoretical_loss": 3.475590591233442,
+      "tokens_seen": 1887240192
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.647889584336383e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.475570402244484,
+      "tokens_seen": 1887371264
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.64708714492056e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.4755502150500863,
+      "tokens_seen": 1887502336
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.646284705504735e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4755300296499625,
+      "tokens_seen": 1887633408
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.645482266088911e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.4755098460438307,
+      "tokens_seen": 1887764480
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.644679826673087e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.475489664231406,
+      "tokens_seen": 1887895552
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.643877387257262e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.475469484212404,
+      "tokens_seen": 1888026624
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.643074947841439e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.475449305986542,
+      "tokens_seen": 1888157696
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.642272508425614e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.475429129553535,
+      "tokens_seen": 1888288768
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.64147006900979e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.475408954913101,
+      "tokens_seen": 1888419840
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.640667629593966e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4753887820649547,
+      "tokens_seen": 1888550912
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.639865190178143e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4753686110088124,
+      "tokens_seen": 1888681984
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.639062750762318e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.4753484417443916,
+      "tokens_seen": 1888813056
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.638260311346493e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.475328274271409,
+      "tokens_seen": 1888944128
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.63745787193067e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.47530810858958,
+      "tokens_seen": 1889075200
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.636655432514845e-05,
+      "loss": 1.2921,
+      "theoretical_loss": 3.4752879446986222,
+      "tokens_seen": 1889206272
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.635852993099022e-05,
+      "loss": 1.3293,
+      "theoretical_loss": 3.4752677825982516,
+      "tokens_seen": 1889337344
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.635050553683197e-05,
+      "loss": 1.3381,
+      "theoretical_loss": 3.475247622288186,
+      "tokens_seen": 1889468416
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.634248114267374e-05,
+      "loss": 1.3641,
+      "theoretical_loss": 3.475227463768141,
+      "tokens_seen": 1889599488
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.633445674851549e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4752073070378344,
+      "tokens_seen": 1889730560
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.632643235435724e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.475187152096983,
+      "tokens_seen": 1889861632
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.631840796019901e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.475166998945304,
+      "tokens_seen": 1889992704
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.631038356604076e-05,
+      "loss": 1.2651,
+      "theoretical_loss": 3.475146847582514,
+      "tokens_seen": 1890123776
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.630235917188253e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4751266980083306,
+      "tokens_seen": 1890254848
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.629433477772428e-05,
+      "loss": 1.312,
+      "theoretical_loss": 3.4751065502224705,
+      "tokens_seen": 1890385920
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.628631038356605e-05,
+      "loss": 1.3063,
+      "theoretical_loss": 3.475086404224652,
+      "tokens_seen": 1890516992
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.62782859894078e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.4750662600145916,
+      "tokens_seen": 1890648064
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.627026159524956e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.475046117592007,
+      "tokens_seen": 1890779136
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.626223720109132e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.4750259769566156,
+      "tokens_seen": 1890910208
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.625421280693308e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.4750058381081357,
+      "tokens_seen": 1891041280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.624618841277484e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4749857010462835,
+      "tokens_seen": 1891172352
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.62381640186166e-05,
+      "loss": 1.3194,
+      "theoretical_loss": 3.4749655657707783,
+      "tokens_seen": 1891303424
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.623013962445836e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4749454322813365,
+      "tokens_seen": 1891434496
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.622211523030012e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.4749253005776763,
+      "tokens_seen": 1891565568
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.621409083614188e-05,
+      "loss": 1.2879,
+      "theoretical_loss": 3.4749051706595164,
+      "tokens_seen": 1891696640
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.620606644198364e-05,
+      "loss": 1.295,
+      "theoretical_loss": 3.474885042526574,
+      "tokens_seen": 1891827712
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.619804204782539e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.474864916178566,
+      "tokens_seen": 1891958784
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.619001765366716e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.474844791615213,
+      "tokens_seen": 1892089856
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.618199325950891e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.474824668836231,
+      "tokens_seen": 1892220928
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.617396886535068e-05,
+      "loss": 1.3326,
+      "theoretical_loss": 3.474804547841339,
+      "tokens_seen": 1892352000
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 637400,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2862191200256348,
+      "objective/train/theoretical_loss": 3.4747844286302563,
+      "objective/train/tokens_used": 262942176,
+      "theoretical_loss": 3.4747844286302563,
+      "tokens_seen": 1892483072
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.616594447119243e-05,
+      "loss": 1.3124,
+      "theoretical_loss": 3.4747844286302563,
+      "tokens_seen": 1892483072
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.61579200770342e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.4747643112026996,
+      "tokens_seen": 1892614144
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.614989568287595e-05,
+      "loss": 1.2902,
+      "theoretical_loss": 3.4747441955583875,
+      "tokens_seen": 1892745216
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.61418712887177e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.474724081697039,
+      "tokens_seen": 1892876288
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.613384689455947e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4747039696183726,
+      "tokens_seen": 1893007360
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.612582250040122e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.4746838593221065,
+      "tokens_seen": 1893138432
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.611779810624299e-05,
+      "loss": 1.3112,
+      "theoretical_loss": 3.4746637508079603,
+      "tokens_seen": 1893269504
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.610977371208474e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.4746436440756514,
+      "tokens_seen": 1893400576
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.610174931792651e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.474623539124899,
+      "tokens_seen": 1893531648
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.609372492376826e-05,
+      "loss": 1.3135,
+      "theoretical_loss": 3.474603435955423,
+      "tokens_seen": 1893662720
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.608570052961001e-05,
+      "loss": 1.2976,
+      "theoretical_loss": 3.474583334566941,
+      "tokens_seen": 1893793792
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.607767613545178e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.474563234959172,
+      "tokens_seen": 1893924864
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.606965174129353e-05,
+      "loss": 1.341,
+      "theoretical_loss": 3.474543137131836,
+      "tokens_seen": 1894055936
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.60616273471353e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.474523041084651,
+      "tokens_seen": 1894187008
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.605360295297705e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.4745029468173367,
+      "tokens_seen": 1894318080
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.604557855881882e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.474482854329613,
+      "tokens_seen": 1894449152
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.603755416466057e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.474462763621198,
+      "tokens_seen": 1894580224
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.602952977050233e-05,
+      "loss": 1.311,
+      "theoretical_loss": 3.4744426746918116,
+      "tokens_seen": 1894711296
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.60215053763441e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.474422587541173,
+      "tokens_seen": 1894842368
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.601348098218585e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.474402502169002,
+      "tokens_seen": 1894973440
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.600545658802761e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.474382418575018,
+      "tokens_seen": 1895104512
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.599743219386937e-05,
+      "loss": 1.1791,
+      "theoretical_loss": 3.474362336758941,
+      "tokens_seen": 1895235584
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.598940779971113e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.47434225672049,
+      "tokens_seen": 1895366656
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.598138340555289e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.4743221784593845,
+      "tokens_seen": 1895497728
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.597335901139464e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.4743021019753453,
+      "tokens_seen": 1895628800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.59653346172364e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.4742820272680914,
+      "tokens_seen": 1895759872
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.595731022307816e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4742619543373436,
+      "tokens_seen": 1895890944
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.594928582891993e-05,
+      "loss": 1.3158,
+      "theoretical_loss": 3.4742418831828212,
+      "tokens_seen": 1896022016
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.594126143476168e-05,
+      "loss": 1.3079,
+      "theoretical_loss": 3.474221813804244,
+      "tokens_seen": 1896153088
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.593323704060345e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.4742017462013326,
+      "tokens_seen": 1896284160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.59252126464452e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4741816803738077,
+      "tokens_seen": 1896415232
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.591718825228697e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.474161616321388,
+      "tokens_seen": 1896546304
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.590916385812872e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.474141554043795,
+      "tokens_seen": 1896677376
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.590113946397047e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.47412149354075,
+      "tokens_seen": 1896808448
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.589311506981224e-05,
+      "loss": 1.3089,
+      "theoretical_loss": 3.474101434811971,
+      "tokens_seen": 1896939520
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.588509067565399e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.47408137785718,
+      "tokens_seen": 1897070592
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.587706628149576e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.4740613226760972,
+      "tokens_seen": 1897201664
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.586904188733751e-05,
+      "loss": 1.3346,
+      "theoretical_loss": 3.4740412692684437,
+      "tokens_seen": 1897332736
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.586101749317928e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.47402121763394,
+      "tokens_seen": 1897463808
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.585299309902103e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.4740011677723057,
+      "tokens_seen": 1897594880
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.584496870486278e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4739811196832626,
+      "tokens_seen": 1897725952
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.583694431070455e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.4739610733665325,
+      "tokens_seen": 1897857024
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.58289199165463e-05,
+      "loss": 1.3222,
+      "theoretical_loss": 3.4739410288218346,
+      "tokens_seen": 1897988096
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.582089552238807e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.473920986048891,
+      "tokens_seen": 1898119168
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.581287112822982e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.473900945047422,
+      "tokens_seen": 1898250240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.580484673407159e-05,
+      "loss": 1.2808,
+      "theoretical_loss": 3.4738809058171496,
+      "tokens_seen": 1898381312
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.579682233991334e-05,
+      "loss": 1.3433,
+      "theoretical_loss": 3.4738608683577947,
+      "tokens_seen": 1898512384
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.57887979457551e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.473840832669078,
+      "tokens_seen": 1898643456
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.578077355159686e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.4738207987507215,
+      "tokens_seen": 1898774528
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.577274915743862e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.473800766602446,
+      "tokens_seen": 1898905600
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 639521,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2776257991790771,
+      "objective/train/theoretical_loss": 3.4737807362239734,
+      "objective/train/tokens_used": 269495776,
+      "theoretical_loss": 3.4737807362239734,
+      "tokens_seen": 1899036672
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.576472476328038e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.4737807362239734,
+      "tokens_seen": 1899036672
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.575670036912214e-05,
+      "loss": 1.326,
+      "theoretical_loss": 3.473760707615025,
+      "tokens_seen": 1899167744
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.57486759749639e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4737406807753226,
+      "tokens_seen": 1899298816
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.574065158080566e-05,
+      "loss": 1.32,
+      "theoretical_loss": 3.4737206557045877,
+      "tokens_seen": 1899429888
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.573262718664741e-05,
+      "loss": 1.3224,
+      "theoretical_loss": 3.4737006324025415,
+      "tokens_seen": 1899560960
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.572460279248918e-05,
+      "loss": 1.302,
+      "theoretical_loss": 3.4736806108689064,
+      "tokens_seen": 1899692032
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.571657839833093e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.473660591103404,
+      "tokens_seen": 1899823104
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.57085540041727e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.4736405731057562,
+      "tokens_seen": 1899954176
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.570052961001445e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.473620556875685,
+      "tokens_seen": 1900085248
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.569250521585622e-05,
+      "loss": 1.3317,
+      "theoretical_loss": 3.473600542412912,
+      "tokens_seen": 1900216320
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.568448082169797e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.47358052971716,
+      "tokens_seen": 1900347392
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.567645642753972e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.473560518788151,
+      "tokens_seen": 1900478464
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.566843203338149e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.4735405096256065,
+      "tokens_seen": 1900609536
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.566040763922324e-05,
+      "loss": 1.3251,
+      "theoretical_loss": 3.4735205022292495,
+      "tokens_seen": 1900740608
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.565238324506501e-05,
+      "loss": 1.2947,
+      "theoretical_loss": 3.4735004965988017,
+      "tokens_seen": 1900871680
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.564435885090676e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.473480492733986,
+      "tokens_seen": 1901002752
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.563633445674853e-05,
+      "loss": 1.3099,
+      "theoretical_loss": 3.4734604906345248,
+      "tokens_seen": 1901133824
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.562831006259028e-05,
+      "loss": 1.2892,
+      "theoretical_loss": 3.47344049030014,
+      "tokens_seen": 1901264896
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.562028566843204e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.4734204917305553,
+      "tokens_seen": 1901395968
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.56122612742738e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.4734004949254924,
+      "tokens_seen": 1901527040
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.560423688011555e-05,
+      "loss": 1.3534,
+      "theoretical_loss": 3.4733804998846747,
+      "tokens_seen": 1901658112
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.559621248595732e-05,
+      "loss": 1.3296,
+      "theoretical_loss": 3.4733605066078237,
+      "tokens_seen": 1901789184
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.558818809179907e-05,
+      "loss": 1.3018,
+      "theoretical_loss": 3.473340515094664,
+      "tokens_seen": 1901920256
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.558016369764084e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.473320525344917,
+      "tokens_seen": 1902051328
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.55721393034826e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4733005373583064,
+      "tokens_seen": 1902182400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.556411490932436e-05,
+      "loss": 1.2113,
+      "theoretical_loss": 3.473280551134555,
+      "tokens_seen": 1902313472
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.555609051516611e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.473260566673386,
+      "tokens_seen": 1902444544
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.554806612100787e-05,
+      "loss": 1.328,
+      "theoretical_loss": 3.473240583974522,
+      "tokens_seen": 1902575616
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.554004172684963e-05,
+      "loss": 1.2903,
+      "theoretical_loss": 3.473220603037687,
+      "tokens_seen": 1902706688
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.553201733269139e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.473200623862604,
+      "tokens_seen": 1902837760
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.552399293853315e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.4731806464489963,
+      "tokens_seen": 1902968832
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.551596854437491e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.473160670796587,
+      "tokens_seen": 1903099904
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.550794415021667e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.4731406969050997,
+      "tokens_seen": 1903230976
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.549991975605843e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.473120724774258,
+      "tokens_seen": 1903362048
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.549189536190018e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.473100754403786,
+      "tokens_seen": 1903493120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.548387096774195e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.4730807857934063,
+      "tokens_seen": 1903624192
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.54758465735837e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.4730608189428436,
+      "tokens_seen": 1903755264
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.546782217942547e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.4730408538518205,
+      "tokens_seen": 1903886336
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.545979778526722e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.473020890520062,
+      "tokens_seen": 1904017408
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.545177339110899e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4730009289472914,
+      "tokens_seen": 1904148480
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.544374899695074e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.472980969133232,
+      "tokens_seen": 1904279552
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.543572460279249e-05,
+      "loss": 1.2964,
+      "theoretical_loss": 3.4729610110776097,
+      "tokens_seen": 1904410624
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.542770020863426e-05,
+      "loss": 1.2957,
+      "theoretical_loss": 3.4729410547801463,
+      "tokens_seen": 1904541696
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.541967581447601e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.472921100240568,
+      "tokens_seen": 1904672768
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.541165142031778e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4729011474585976,
+      "tokens_seen": 1904803840
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.540362702615953e-05,
+      "loss": 1.3682,
+      "theoretical_loss": 3.47288119643396,
+      "tokens_seen": 1904934912
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.53956026320013e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.472861247166379,
+      "tokens_seen": 1905065984
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.538757823784305e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.4728412996555797,
+      "tokens_seen": 1905197056
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.53795538436848e-05,
+      "loss": 1.3176,
+      "theoretical_loss": 3.4728213539012858,
+      "tokens_seen": 1905328128
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.537152944952657e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.472801409903223,
+      "tokens_seen": 1905459200
+    },
+    {
+      "epoch": 0.15,
+      "objective/train/docs_used": 641709,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.319300651550293,
+      "objective/train/theoretical_loss": 3.472781467661114,
+      "objective/train/tokens_used": 276049376,
+      "theoretical_loss": 3.472781467661114,
+      "tokens_seen": 1905590272
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.536350505536833e-05,
+      "loss": 1.3453,
+      "theoretical_loss": 3.472781467661114,
+      "tokens_seen": 1905590272
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.535548066121009e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.472761527174685,
+      "tokens_seen": 1905721344
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.534745626705184e-05,
+      "loss": 1.3236,
+      "theoretical_loss": 3.47274158844366,
+      "tokens_seen": 1905852416
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.533943187289361e-05,
+      "loss": 1.3578,
+      "theoretical_loss": 3.472721651467764,
+      "tokens_seen": 1905983488
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.533140747873536e-05,
+      "loss": 1.3305,
+      "theoretical_loss": 3.472701716246722,
+      "tokens_seen": 1906114560
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.532338308457712e-05,
+      "loss": 1.3513,
+      "theoretical_loss": 3.472681782780259,
+      "tokens_seen": 1906245632
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.531535869041888e-05,
+      "loss": 1.3822,
+      "theoretical_loss": 3.472661851068099,
+      "tokens_seen": 1906376704
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.530733429626064e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4726419211099677,
+      "tokens_seen": 1906507776
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.52993099021024e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.47262199290559,
+      "tokens_seen": 1906638848
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.529128550794416e-05,
+      "loss": 1.3135,
+      "theoretical_loss": 3.4726020664546913,
+      "tokens_seen": 1906769920
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.528326111378591e-05,
+      "loss": 1.3273,
+      "theoretical_loss": 3.4725821417569964,
+      "tokens_seen": 1906900992
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.527523671962768e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.472562218812231,
+      "tokens_seen": 1907032064
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.526721232546943e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.47254229762012,
+      "tokens_seen": 1907163136
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.52591879313112e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.472522378180389,
+      "tokens_seen": 1907294208
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.525116353715295e-05,
+      "loss": 1.3189,
+      "theoretical_loss": 3.472502460492763,
+      "tokens_seen": 1907425280
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.524313914299472e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.4724825445569687,
+      "tokens_seen": 1907556352
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.523511474883647e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.47246263037273,
+      "tokens_seen": 1907687424
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.522709035467822e-05,
+      "loss": 1.3082,
+      "theoretical_loss": 3.4724427179397734,
+      "tokens_seen": 1907818496
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.521906596051999e-05,
+      "loss": 1.3124,
+      "theoretical_loss": 3.4724228072578254,
+      "tokens_seen": 1907949568
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.521104156636174e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.47240289832661,
+      "tokens_seen": 1908080640
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.520301717220351e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.472382991145855,
+      "tokens_seen": 1908211712
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.519499277804526e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.4723630857152843,
+      "tokens_seen": 1908342784
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.518696838388702e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.472343182034625,
+      "tokens_seen": 1908473856
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.517894398972878e-05,
+      "loss": 1.3254,
+      "theoretical_loss": 3.472323280103603,
+      "tokens_seen": 1908604928
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.517091959557054e-05,
+      "loss": 1.3694,
+      "theoretical_loss": 3.472303379921944,
+      "tokens_seen": 1908736000
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.51628952014123e-05,
+      "loss": 1.2989,
+      "theoretical_loss": 3.472283481489374,
+      "tokens_seen": 1908867072
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.515487080725406e-05,
+      "loss": 1.3229,
+      "theoretical_loss": 3.47226358480562,
+      "tokens_seen": 1908998144
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.514684641309582e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4722436898704077,
+      "tokens_seen": 1909129216
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.513882201893758e-05,
+      "loss": 1.2808,
+      "theoretical_loss": 3.4722237966834633,
+      "tokens_seen": 1909260288
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.513079762477933e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.4722039052445135,
+      "tokens_seen": 1909391360
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.51227732306211e-05,
+      "loss": 1.3444,
+      "theoretical_loss": 3.4721840155532844,
+      "tokens_seen": 1909522432
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.511474883646285e-05,
+      "loss": 1.3191,
+      "theoretical_loss": 3.4721641276095023,
+      "tokens_seen": 1909653504
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.510672444230461e-05,
+      "loss": 1.3473,
+      "theoretical_loss": 3.4721442414128942,
+      "tokens_seen": 1909784576
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.509870004814637e-05,
+      "loss": 1.3811,
+      "theoretical_loss": 3.472124356963187,
+      "tokens_seen": 1909915648
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.509067565398812e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4721044742601066,
+      "tokens_seen": 1910046720
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.508265125982989e-05,
+      "loss": 1.2947,
+      "theoretical_loss": 3.4720845933033804,
+      "tokens_seen": 1910177792
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.507462686567164e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4720647140927348,
+      "tokens_seen": 1910308864
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.506660247151341e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.4720448366278966,
+      "tokens_seen": 1910439936
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.505857807735516e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.472024960908593,
+      "tokens_seen": 1910571008
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.505055368319693e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4720050869345513,
+      "tokens_seen": 1910702080
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.504252928903868e-05,
+      "loss": 1.3484,
+      "theoretical_loss": 3.471985214705497,
+      "tokens_seen": 1910833152
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.503450489488043e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.471965344221159,
+      "tokens_seen": 1910964224
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.50264805007222e-05,
+      "loss": 1.3083,
+      "theoretical_loss": 3.471945475481264,
+      "tokens_seen": 1911095296
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.501845610656395e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.471925608485539,
+      "tokens_seen": 1911226368
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.501043171240572e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.471905743233711,
+      "tokens_seen": 1911357440
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.500240731824747e-05,
+      "loss": 1.3799,
+      "theoretical_loss": 3.471885879725508,
+      "tokens_seen": 1911488512
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.499438292408923e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.4718660179606564,
+      "tokens_seen": 1911619584
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.498635852993099e-05,
+      "loss": 1.3254,
+      "theoretical_loss": 3.471846157938885,
+      "tokens_seen": 1911750656
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.497833413577275e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.47182629965992,
+      "tokens_seen": 1911881728
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.497030974161451e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4718064431234903,
+      "tokens_seen": 1912012800
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 643869,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3385043144226074,
+      "objective/train/theoretical_loss": 3.471786588329323,
+      "objective/train/tokens_used": 282602976,
+      "theoretical_loss": 3.471786588329323,
+      "tokens_seen": 1912143872
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.496228534745627e-05,
+      "loss": 1.3221,
+      "theoretical_loss": 3.471786588329323,
+      "tokens_seen": 1912143872
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.495426095329802e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.471766735277145,
+      "tokens_seen": 1912274944
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.494623655913979e-05,
+      "loss": 1.3377,
+      "theoretical_loss": 3.471746883966685,
+      "tokens_seen": 1912406016
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.493821216498154e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.471727034397671,
+      "tokens_seen": 1912537088
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.49301877708233e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.471707186569831,
+      "tokens_seen": 1912668160
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.492216337666506e-05,
+      "loss": 1.2913,
+      "theoretical_loss": 3.471687340482892,
+      "tokens_seen": 1912799232
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.491413898250683e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.4716674961365825,
+      "tokens_seen": 1912930304
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.490611458834858e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.4716476535306304,
+      "tokens_seen": 1913061376
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.489809019419033e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.4716278126647646,
+      "tokens_seen": 1913192448
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.48900658000321e-05,
+      "loss": 1.3057,
+      "theoretical_loss": 3.4716079735387124,
+      "tokens_seen": 1913323520
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.488204140587385e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.471588136152202,
+      "tokens_seen": 1913454592
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.487401701171562e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.471568300504963,
+      "tokens_seen": 1913585664
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.486599261755737e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.471548466596723,
+      "tokens_seen": 1913716736
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.485796822339914e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.47152863442721,
+      "tokens_seen": 1913847808
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.484994382924089e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.4715088039961532,
+      "tokens_seen": 1913978880
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.484191943508264e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4714889753032807,
+      "tokens_seen": 1914109952
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.483389504092441e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4714691483483215,
+      "tokens_seen": 1914241024
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.482587064676616e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4714493231310044,
+      "tokens_seen": 1914372096
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.481784625260793e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.471429499651057,
+      "tokens_seen": 1914503168
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.480982185844968e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.47140967790821,
+      "tokens_seen": 1914634240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.480179746429145e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.471389857902191,
+      "tokens_seen": 1914765312
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.47937730701332e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.471370039632729,
+      "tokens_seen": 1914896384
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.478574867597496e-05,
+      "loss": 1.3808,
+      "theoretical_loss": 3.471350223099553,
+      "tokens_seen": 1915027456
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.477772428181672e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.471330408302393,
+      "tokens_seen": 1915158528
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.476969988765848e-05,
+      "loss": 1.3323,
+      "theoretical_loss": 3.4713105952409773,
+      "tokens_seen": 1915289600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.476167549350024e-05,
+      "loss": 1.3448,
+      "theoretical_loss": 3.471290783915035,
+      "tokens_seen": 1915420672
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.4753651099342e-05,
+      "loss": 1.1865,
+      "theoretical_loss": 3.471270974324295,
+      "tokens_seen": 1915551744
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.474562670518376e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.4712511664684875,
+      "tokens_seen": 1915682816
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.473760231102552e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4712313603473417,
+      "tokens_seen": 1915813888
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.472957791686727e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.471211555960587,
+      "tokens_seen": 1915944960
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.472155352270904e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4711917533079526,
+      "tokens_seen": 1916076032
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.471352912855079e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.4711719523891684,
+      "tokens_seen": 1916207104
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.470550473439256e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.4711521532039633,
+      "tokens_seen": 1916338176
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.469748034023431e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4711323557520677,
+      "tokens_seen": 1916469248
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.468945594607608e-05,
+      "loss": 1.2777,
+      "theoretical_loss": 3.4711125600332107,
+      "tokens_seen": 1916600320
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.468143155191783e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.4710927660471227,
+      "tokens_seen": 1916731392
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.467340715775958e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.4710729737935333,
+      "tokens_seen": 1916862464
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.466538276360135e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.4710531832721725,
+      "tokens_seen": 1916993536
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.46573583694431e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4710333944827703,
+      "tokens_seen": 1917124608
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.464933397528487e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.471013607425056,
+      "tokens_seen": 1917255680
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.464130958112662e-05,
+      "loss": 1.3019,
+      "theoretical_loss": 3.470993822098761,
+      "tokens_seen": 1917386752
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.463328518696839e-05,
+      "loss": 1.3374,
+      "theoretical_loss": 3.4709740385036145,
+      "tokens_seen": 1917517824
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.462526079281014e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.470954256639347,
+      "tokens_seen": 1917648896
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.461723639865191e-05,
+      "loss": 1.3029,
+      "theoretical_loss": 3.470934476505689,
+      "tokens_seen": 1917779968
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.460921200449366e-05,
+      "loss": 1.2734,
+      "theoretical_loss": 3.47091469810237,
+      "tokens_seen": 1917911040
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.460118761033541e-05,
+      "loss": 1.3065,
+      "theoretical_loss": 3.470894921429122,
+      "tokens_seen": 1918042112
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.459316321617718e-05,
+      "loss": 1.2822,
+      "theoretical_loss": 3.4708751464856737,
+      "tokens_seen": 1918173184
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.458513882201893e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.470855373271756,
+      "tokens_seen": 1918304256
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.45771144278607e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.4708356017871003,
+      "tokens_seen": 1918435328
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.456909003370245e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.4708158320314366,
+      "tokens_seen": 1918566400
+    },
+    {
+      "epoch": 0.16,
+      "objective/train/docs_used": 645829,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.193943977355957,
+      "objective/train/theoretical_loss": 3.470796064004496,
+      "objective/train/tokens_used": 289156576,
+      "theoretical_loss": 3.470796064004496,
+      "tokens_seen": 1918697472
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.456106563954422e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.470796064004496,
+      "tokens_seen": 1918697472
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.455304124538597e-05,
+      "loss": 1.2812,
+      "theoretical_loss": 3.470776297706009,
+      "tokens_seen": 1918828544
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.454501685122773e-05,
+      "loss": 1.2727,
+      "theoretical_loss": 3.4707565331357064,
+      "tokens_seen": 1918959616
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.45369924570695e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4707367702933194,
+      "tokens_seen": 1919090688
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.452896806291125e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.470717009178578,
+      "tokens_seen": 1919221760
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.452094366875301e-05,
+      "loss": 1.3011,
+      "theoretical_loss": 3.470697249791214,
+      "tokens_seen": 1919352832
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.451291927459477e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.4706774921309584,
+      "tokens_seen": 1919483904
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.450489488043653e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.4706577361975426,
+      "tokens_seen": 1919614976
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.449687048627829e-05,
+      "loss": 1.289,
+      "theoretical_loss": 3.4706379819906967,
+      "tokens_seen": 1919746048
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.448884609212004e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.470618229510153,
+      "tokens_seen": 1919877120
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.44808216979618e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.470598478755643,
+      "tokens_seen": 1920008192
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.447279730380356e-05,
+      "loss": 1.2885,
+      "theoretical_loss": 3.4705787297268964,
+      "tokens_seen": 1920139264
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.446477290964533e-05,
+      "loss": 1.3405,
+      "theoretical_loss": 3.470558982423647,
+      "tokens_seen": 1920270336
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.445674851548708e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.470539236845624,
+      "tokens_seen": 1920401408
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.444872412132885e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.47051949299256,
+      "tokens_seen": 1920532480
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.44406997271706e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.470499750864187,
+      "tokens_seen": 1920663552
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.443267533301235e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.470480010460236,
+      "tokens_seen": 1920794624
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.442465093885412e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.470460271780439,
+      "tokens_seen": 1920925696
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.441662654469587e-05,
+      "loss": 1.314,
+      "theoretical_loss": 3.4704405348245277,
+      "tokens_seen": 1921056768
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.440860215053764e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.4704207995922336,
+      "tokens_seen": 1921187840
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.440057775637939e-05,
+      "loss": 1.3326,
+      "theoretical_loss": 3.470401066083289,
+      "tokens_seen": 1921318912
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.439255336222116e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.470381334297426,
+      "tokens_seen": 1921449984
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.438452896806291e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.470361604234377,
+      "tokens_seen": 1921581056
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.437650457390466e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4703418758938724,
+      "tokens_seen": 1921712128
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.436848017974643e-05,
+      "loss": 1.2947,
+      "theoretical_loss": 3.470322149275646,
+      "tokens_seen": 1921843200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.436045578558818e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4703024243794296,
+      "tokens_seen": 1921974272
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.435243139142995e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.4702827012049546,
+      "tokens_seen": 1922105344
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.43444069972717e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.470262979751955,
+      "tokens_seen": 1922236416
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.433638260311347e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.4702432600201614,
+      "tokens_seen": 1922367488
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.432835820895522e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.4702235420093075,
+      "tokens_seen": 1922498560
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.432033381479699e-05,
+      "loss": 1.3443,
+      "theoretical_loss": 3.470203825719125,
+      "tokens_seen": 1922629632
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.431230942063874e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.470184111149347,
+      "tokens_seen": 1922760704
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.43042850264805e-05,
+      "loss": 1.338,
+      "theoretical_loss": 3.4701643982997057,
+      "tokens_seen": 1922891776
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.429626063232226e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4701446871699346,
+      "tokens_seen": 1923022848
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.428823623816402e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.470124977759765,
+      "tokens_seen": 1923153920
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.428021184400578e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.470105270068931,
+      "tokens_seen": 1923284992
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.427218744984754e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4700855640971646,
+      "tokens_seen": 1923416064
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.42641630556893e-05,
+      "loss": 1.1653,
+      "theoretical_loss": 3.4700658598442,
+      "tokens_seen": 1923547136
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.425613866153106e-05,
+      "loss": 1.3332,
+      "theoretical_loss": 3.4700461573097687,
+      "tokens_seen": 1923678208
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.424811426737281e-05,
+      "loss": 1.3233,
+      "theoretical_loss": 3.470026456493604,
+      "tokens_seen": 1923809280
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.424008987321458e-05,
+      "loss": 1.3087,
+      "theoretical_loss": 3.4700067573954394,
+      "tokens_seen": 1923940352
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.423206547905633e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4699870600150087,
+      "tokens_seen": 1924071424
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.42240410848981e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.4699673643520437,
+      "tokens_seen": 1924202496
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.421601669073985e-05,
+      "loss": 1.2106,
+      "theoretical_loss": 3.4699476704062784,
+      "tokens_seen": 1924333568
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.420799229658162e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.469927978177446,
+      "tokens_seen": 1924464640
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.419996790242337e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4699082876652803,
+      "tokens_seen": 1924595712
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.419194350826512e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.4698885988695145,
+      "tokens_seen": 1924726784
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.418391911410689e-05,
+      "loss": 1.2828,
+      "theoretical_loss": 3.4698689117898818,
+      "tokens_seen": 1924857856
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.417589471994864e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.469849226426116,
+      "tokens_seen": 1924988928
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.416787032579041e-05,
+      "loss": 1.2078,
+      "theoretical_loss": 3.4698295427779513,
+      "tokens_seen": 1925120000
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 648016,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.235392689704895,
+      "objective/train/theoretical_loss": 3.46980986084512,
+      "objective/train/tokens_used": 295710176,
+      "theoretical_loss": 3.46980986084512,
+      "tokens_seen": 1925251072
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.415984593163216e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.46980986084512,
+      "tokens_seen": 1925251072
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.415182153747393e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.4697901806273572,
+      "tokens_seen": 1925382144
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.414379714331568e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4697705021243963,
+      "tokens_seen": 1925513216
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.413577274915743e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.469750825335971,
+      "tokens_seen": 1925644288
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.41277483549992e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.4697311502618153,
+      "tokens_seen": 1925775360
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.411972396084095e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.4697114769016633,
+      "tokens_seen": 1925906432
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.411169956668272e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.469691805255249,
+      "tokens_seen": 1926037504
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.410367517252447e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4696721353223063,
+      "tokens_seen": 1926168576
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.409565077836624e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.469652467102569,
+      "tokens_seen": 1926299648
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.4087626384208e-05,
+      "loss": 1.32,
+      "theoretical_loss": 3.4696328005957726,
+      "tokens_seen": 1926430720
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.407960199004975e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.4696131358016506,
+      "tokens_seen": 1926561792
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.407157759589151e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.469593472719937,
+      "tokens_seen": 1926692864
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.406355320173327e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.469573811350367,
+      "tokens_seen": 1926823936
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.405552880757503e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.469554151692674,
+      "tokens_seen": 1926955008
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.404750441341679e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.469534493746594,
+      "tokens_seen": 1927086080
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.403948001925855e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4695148375118605,
+      "tokens_seen": 1927217152
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.403145562510031e-05,
+      "loss": 1.3033,
+      "theoretical_loss": 3.469495182988208,
+      "tokens_seen": 1927348224
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.402343123094206e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.469475530175372,
+      "tokens_seen": 1927479296
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.401540683678383e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4694558790730867,
+      "tokens_seen": 1927610368
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.400738244262558e-05,
+      "loss": 1.3035,
+      "theoretical_loss": 3.4694362296810874,
+      "tokens_seen": 1927741440
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.399935804846735e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4694165819991083,
+      "tokens_seen": 1927872512
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.39913336543091e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.469396936026885,
+      "tokens_seen": 1928003584
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.398330926015087e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.469377291764152,
+      "tokens_seen": 1928134656
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.397528486599262e-05,
+      "loss": 1.2349,
+      "theoretical_loss": 3.4693576492106444,
+      "tokens_seen": 1928265728
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.396726047183439e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.4693380083660976,
+      "tokens_seen": 1928396800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.395923607767614e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4693183692302467,
+      "tokens_seen": 1928527872
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.395121168351789e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.4692987318028266,
+      "tokens_seen": 1928658944
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.394318728935966e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.4692790960835733,
+      "tokens_seen": 1928790016
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.393516289520141e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4692594620722215,
+      "tokens_seen": 1928921088
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.392713850104318e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4692398297685063,
+      "tokens_seen": 1929052160
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.391911410688493e-05,
+      "loss": 1.3016,
+      "theoretical_loss": 3.4692201991721645,
+      "tokens_seen": 1929183232
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.39110897127267e-05,
+      "loss": 1.3446,
+      "theoretical_loss": 3.46920057028293,
+      "tokens_seen": 1929314304
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.390306531856845e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.46918094310054,
+      "tokens_seen": 1929445376
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.38950409244102e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.469161317624729,
+      "tokens_seen": 1929576448
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.388701653025197e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4691416938552333,
+      "tokens_seen": 1929707520
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.387899213609372e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.469122071791788,
+      "tokens_seen": 1929838592
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.387096774193549e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.4691024514341295,
+      "tokens_seen": 1929969664
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.386294334777724e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.469082832781994,
+      "tokens_seen": 1930100736
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.385491895361901e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.4690632158351162,
+      "tokens_seen": 1930231808
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.384689455946076e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.469043600593233,
+      "tokens_seen": 1930362880
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.383887016530252e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4690239870560804,
+      "tokens_seen": 1930493952
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.383084577114428e-05,
+      "loss": 1.374,
+      "theoretical_loss": 3.4690043752233946,
+      "tokens_seen": 1930625024
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.382282137698604e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.4689847650949117,
+      "tokens_seen": 1930756096
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.38147969828278e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.4689651566703676,
+      "tokens_seen": 1930887168
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.380677258866956e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.468945549949499,
+      "tokens_seen": 1931018240
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.379874819451132e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4689259449320415,
+      "tokens_seen": 1931149312
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.379072380035308e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.468906341617733,
+      "tokens_seen": 1931280384
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.378269940619483e-05,
+      "loss": 1.3141,
+      "theoretical_loss": 3.468886740006308,
+      "tokens_seen": 1931411456
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.37746750120366e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.468867140097505,
+      "tokens_seen": 1931542528
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.376665061787835e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.4688475418910594,
+      "tokens_seen": 1931673600
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 650108,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.24700927734375,
+      "objective/train/theoretical_loss": 3.4688279453867077,
+      "objective/train/tokens_used": 302263776,
+      "theoretical_loss": 3.4688279453867077,
+      "tokens_seen": 1931804672
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.375862622372012e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4688279453867077,
+      "tokens_seen": 1931804672
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.375060182956187e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.4688083505841876,
+      "tokens_seen": 1931935744
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.374257743540364e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.4687887574832352,
+      "tokens_seen": 1932066816
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.373455304124539e-05,
+      "loss": 1.3668,
+      "theoretical_loss": 3.468769166083587,
+      "tokens_seen": 1932197888
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.372652864708714e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.4687495763849805,
+      "tokens_seen": 1932328960
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.371850425292891e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.4687299883871523,
+      "tokens_seen": 1932460032
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.371047985877066e-05,
+      "loss": 1.3344,
+      "theoretical_loss": 3.46871040208984,
+      "tokens_seen": 1932591104
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.370245546461243e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.4686908174927797,
+      "tokens_seen": 1932722176
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.369443107045418e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.4686712345957087,
+      "tokens_seen": 1932853248
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.368640667629595e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.468651653398365,
+      "tokens_seen": 1932984320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.36783822821377e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.4686320739004852,
+      "tokens_seen": 1933115392
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.367035788797947e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.4686124961018066,
+      "tokens_seen": 1933246464
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.366233349382122e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.468592920002066,
+      "tokens_seen": 1933377536
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.365430909966297e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.468573345601002,
+      "tokens_seen": 1933508608
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.364628470550474e-05,
+      "loss": 1.3273,
+      "theoretical_loss": 3.4685537728983515,
+      "tokens_seen": 1933639680
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.36382603113465e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.4685342018938523,
+      "tokens_seen": 1933770752
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.363023591718826e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.468514632587241,
+      "tokens_seen": 1933901824
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.362221152303001e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4684950649782564,
+      "tokens_seen": 1934032896
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.361418712887178e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.4684754990666358,
+      "tokens_seen": 1934163968
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.360616273471353e-05,
+      "loss": 1.3035,
+      "theoretical_loss": 3.4684559348521162,
+      "tokens_seen": 1934295040
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.359813834055529e-05,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4684363723344367,
+      "tokens_seen": 1934426112
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.359011394639705e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.4684168115133343,
+      "tokens_seen": 1934557184
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.358208955223881e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.468397252388547,
+      "tokens_seen": 1934688256
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.357406515808057e-05,
+      "loss": 1.2848,
+      "theoretical_loss": 3.4683776949598135,
+      "tokens_seen": 1934819328
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.356604076392233e-05,
+      "loss": 1.3276,
+      "theoretical_loss": 3.4683581392268708,
+      "tokens_seen": 1934950400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.35580163697641e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.468338585189458,
+      "tokens_seen": 1935081472
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.354999197560585e-05,
+      "loss": 1.3105,
+      "theoretical_loss": 3.4683190328473126,
+      "tokens_seen": 1935212544
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.35419675814476e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.4682994822001727,
+      "tokens_seen": 1935343616
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.353394318728937e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4682799332477767,
+      "tokens_seen": 1935474688
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.352591879313112e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.468260385989864,
+      "tokens_seen": 1935605760
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.351789439897289e-05,
+      "loss": 1.3046,
+      "theoretical_loss": 3.468240840426171,
+      "tokens_seen": 1935736832
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.350987000481464e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.4682212965564383,
+      "tokens_seen": 1935867904
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.35018456106564e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.468201754380403,
+      "tokens_seen": 1935998976
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.349382121649816e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.468182213897804,
+      "tokens_seen": 1936130048
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.348579682233991e-05,
+      "loss": 1.2983,
+      "theoretical_loss": 3.46816267510838,
+      "tokens_seen": 1936261120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.347777242818168e-05,
+      "loss": 1.3193,
+      "theoretical_loss": 3.46814313801187,
+      "tokens_seen": 1936392192
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.346974803402343e-05,
+      "loss": 1.3103,
+      "theoretical_loss": 3.468123602608012,
+      "tokens_seen": 1936523264
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.34617236398652e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.4681040688965448,
+      "tokens_seen": 1936654336
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.345369924570695e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4680845368772077,
+      "tokens_seen": 1936785408
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.344567485154872e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.4680650065497405,
+      "tokens_seen": 1936916480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.343765045739047e-05,
+      "loss": 1.3007,
+      "theoretical_loss": 3.468045477913881,
+      "tokens_seen": 1937047552
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.342962606323223e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.468025950969368,
+      "tokens_seen": 1937178624
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.342160166907399e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.468006425715941,
+      "tokens_seen": 1937309696
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.341357727491575e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.4679869021533394,
+      "tokens_seen": 1937440768
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.340555288075751e-05,
+      "loss": 1.3691,
+      "theoretical_loss": 3.4679673802813022,
+      "tokens_seen": 1937571840
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.339752848659926e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.467947860099569,
+      "tokens_seen": 1937702912
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.338950409244103e-05,
+      "loss": 1.164,
+      "theoretical_loss": 3.467928341607879,
+      "tokens_seen": 1937833984
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.338147969828278e-05,
+      "loss": 1.3335,
+      "theoretical_loss": 3.4679088248059706,
+      "tokens_seen": 1937965056
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.337345530412455e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.467889309693585,
+      "tokens_seen": 1938096128
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.33654309099663e-05,
+      "loss": 1.3674,
+      "theoretical_loss": 3.46786979627046,
+      "tokens_seen": 1938227200
+    },
+    {
+      "epoch": 0.17,
+      "objective/train/docs_used": 652422,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3212106227874756,
+      "objective/train/theoretical_loss": 3.467850284536336,
+      "objective/train/tokens_used": 308817376,
+      "theoretical_loss": 3.467850284536336,
+      "tokens_seen": 1938358272
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.335740651580806e-05,
+      "loss": 1.2964,
+      "theoretical_loss": 3.467850284536336,
+      "tokens_seen": 1938358272
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.334938212164982e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.467830774490953,
+      "tokens_seen": 1938489344
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.334135772749158e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4678112661340497,
+      "tokens_seen": 1938620416
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.467791759465366,
+      "tokens_seen": 1938751488
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.33253089391751e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.4677722544846428,
+      "tokens_seen": 1938882560
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.331728454501686e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.467752751191619,
+      "tokens_seen": 1939013632
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.330926015085862e-05,
+      "loss": 1.3013,
+      "theoretical_loss": 3.467733249586034,
+      "tokens_seen": 1939144704
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.330123575670037e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.467713749667629,
+      "tokens_seen": 1939275776
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.329321136254214e-05,
+      "loss": 1.3219,
+      "theoretical_loss": 3.467694251436143,
+      "tokens_seen": 1939406848
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.328518696838389e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4676747548913163,
+      "tokens_seen": 1939537920
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.327716257422566e-05,
+      "loss": 1.3014,
+      "theoretical_loss": 3.46765526003289,
+      "tokens_seen": 1939668992
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.326913818006741e-05,
+      "loss": 1.3324,
+      "theoretical_loss": 3.4676357668606026,
+      "tokens_seen": 1939800064
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.326111378590918e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.467616275374196,
+      "tokens_seen": 1939931136
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.325308939175093e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4675967855734093,
+      "tokens_seen": 1940062208
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.324506499759268e-05,
+      "loss": 1.3296,
+      "theoretical_loss": 3.467577297457983,
+      "tokens_seen": 1940193280
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.323704060343445e-05,
+      "loss": 1.3581,
+      "theoretical_loss": 3.4675578110276586,
+      "tokens_seen": 1940324352
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.32290162092762e-05,
+      "loss": 1.3166,
+      "theoretical_loss": 3.467538326282175,
+      "tokens_seen": 1940455424
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.322099181511797e-05,
+      "loss": 1.1795,
+      "theoretical_loss": 3.467518843221274,
+      "tokens_seen": 1940586496
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.321296742095972e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4674993618446956,
+      "tokens_seen": 1940717568
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.320494302680149e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4674798821521806,
+      "tokens_seen": 1940848640
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.319691863264324e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.46746040414347,
+      "tokens_seen": 1940979712
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.3188894238485e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.4674409278183034,
+      "tokens_seen": 1941110784
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.318086984432676e-05,
+      "loss": 1.2816,
+      "theoretical_loss": 3.4674214531764234,
+      "tokens_seen": 1941241856
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.317284545016852e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.467401980217569,
+      "tokens_seen": 1941372928
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.316482105601028e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.4673825089414825,
+      "tokens_seen": 1941504000
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.315679666185203e-05,
+      "loss": 1.3348,
+      "theoretical_loss": 3.4673630393479047,
+      "tokens_seen": 1941635072
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.31487722676938e-05,
+      "loss": 1.1809,
+      "theoretical_loss": 3.467343571436576,
+      "tokens_seen": 1941766144
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.314074787353555e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.467324105207238,
+      "tokens_seen": 1941897216
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.313272347937731e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4673046406596324,
+      "tokens_seen": 1942028288
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.312469908521907e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.4672851777934994,
+      "tokens_seen": 1942159360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.311667469106083e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.4672657166085807,
+      "tokens_seen": 1942290432
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.31086502969026e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.467246257104617,
+      "tokens_seen": 1942421504
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.310062590274435e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.4672267992813515,
+      "tokens_seen": 1942552576
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.309260150858611e-05,
+      "loss": 1.3173,
+      "theoretical_loss": 3.467207343138523,
+      "tokens_seen": 1942683648
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.308457711442787e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.467187888675876,
+      "tokens_seen": 1942814720
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.307655272026962e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.4671684358931496,
+      "tokens_seen": 1942945792
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.306852832611139e-05,
+      "loss": 1.3646,
+      "theoretical_loss": 3.4671489847900867,
+      "tokens_seen": 1943076864
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.306050393195314e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.4671295353664284,
+      "tokens_seen": 1943207936
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.305247953779491e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.4671100876219167,
+      "tokens_seen": 1943339008
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.304445514363666e-05,
+      "loss": 1.2645,
+      "theoretical_loss": 3.4670906415562937,
+      "tokens_seen": 1943470080
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.303643074947843e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4670711971693007,
+      "tokens_seen": 1943601152
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.302840635532018e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.46705175446068,
+      "tokens_seen": 1943732224
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.302038196116195e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.467032313430173,
+      "tokens_seen": 1943863296
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.30123575670037e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.467012874077523,
+      "tokens_seen": 1943994368
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.300433317284545e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.46699343640247,
+      "tokens_seen": 1944125440
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.299630877868722e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.4669740004047584,
+      "tokens_seen": 1944256512
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.298828438452897e-05,
+      "loss": 1.1751,
+      "theoretical_loss": 3.4669545660841283,
+      "tokens_seen": 1944387584
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.298025999037074e-05,
+      "loss": 1.2731,
+      "theoretical_loss": 3.466935133440324,
+      "tokens_seen": 1944518656
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.297223559621249e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.466915702473086,
+      "tokens_seen": 1944649728
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.296421120205426e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.466896273182158,
+      "tokens_seen": 1944780800
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 654435,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1857919692993164,
+      "objective/train/theoretical_loss": 3.466876845567282,
+      "objective/train/tokens_used": 315370976,
+      "theoretical_loss": 3.466876845567282,
+      "tokens_seen": 1944911872
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.295618680789601e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.466876845567282,
+      "tokens_seen": 1944911872
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.294816241373777e-05,
+      "loss": 1.3219,
+      "theoretical_loss": 3.4668574196282,
+      "tokens_seen": 1945042944
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.294013801957953e-05,
+      "loss": 1.1482,
+      "theoretical_loss": 3.4668379953646546,
+      "tokens_seen": 1945174016
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.293211362542129e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.466818572776389,
+      "tokens_seen": 1945305088
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.292408923126305e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.4667991518631456,
+      "tokens_seen": 1945436160
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.29160648371048e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.466779732624667,
+      "tokens_seen": 1945567232
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.290804044294657e-05,
+      "loss": 1.3361,
+      "theoretical_loss": 3.4667603150606956,
+      "tokens_seen": 1945698304
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.290001604878832e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4667408991709756,
+      "tokens_seen": 1945829376
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.289199165463008e-05,
+      "loss": 1.1774,
+      "theoretical_loss": 3.466721484955248,
+      "tokens_seen": 1945960448
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.288396726047184e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.4667020724132573,
+      "tokens_seen": 1946091520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.28759428663136e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.4666826615447452,
+      "tokens_seen": 1946222592
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.286791847215536e-05,
+      "loss": 1.2755,
+      "theoretical_loss": 3.4666632523494556,
+      "tokens_seen": 1946353664
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.285989407799712e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.4666438448271313,
+      "tokens_seen": 1946484736
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.285186968383888e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.4666244389775156,
+      "tokens_seen": 1946615808
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.284384528968064e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.466605034800352,
+      "tokens_seen": 1946746880
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.283582089552239e-05,
+      "loss": 1.3033,
+      "theoretical_loss": 3.466585632295383,
+      "tokens_seen": 1946877952
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.282779650136416e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.4665662314623527,
+      "tokens_seen": 1947009024
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.281977210720591e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.4665468323010034,
+      "tokens_seen": 1947140096
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.281174771304768e-05,
+      "loss": 1.2855,
+      "theoretical_loss": 3.4665274348110797,
+      "tokens_seen": 1947271168
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.280372331888943e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.466508038992325,
+      "tokens_seen": 1947402240
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.27956989247312e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.4664886448444823,
+      "tokens_seen": 1947533312
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.278767453057295e-05,
+      "loss": 1.1374,
+      "theoretical_loss": 3.4664692523672955,
+      "tokens_seen": 1947664384
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.27796501364147e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.466449861560508,
+      "tokens_seen": 1947795456
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.277162574225647e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.4664304724238644,
+      "tokens_seen": 1947926528
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.276360134809822e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.466411084957107,
+      "tokens_seen": 1948057600
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.275557695393999e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.4663916991599812,
+      "tokens_seen": 1948188672
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.274755255978174e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4663723150322294,
+      "tokens_seen": 1948319744
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.273952816562351e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.4663529325735967,
+      "tokens_seen": 1948450816
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.273150377146526e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.466333551783827,
+      "tokens_seen": 1948581888
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.272347937730703e-05,
+      "loss": 1.3732,
+      "theoretical_loss": 3.466314172662664,
+      "tokens_seen": 1948712960
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.271545498314878e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4662947952098517,
+      "tokens_seen": 1948844032
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.270743058899054e-05,
+      "loss": 1.3259,
+      "theoretical_loss": 3.4662754194251346,
+      "tokens_seen": 1948975104
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.26994061948323e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.466256045308257,
+      "tokens_seen": 1949106176
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.269138180067406e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.4662366728589626,
+      "tokens_seen": 1949237248
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.268335740651582e-05,
+      "loss": 1.3179,
+      "theoretical_loss": 3.4662173020769966,
+      "tokens_seen": 1949368320
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.267533301235758e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.466197932962103,
+      "tokens_seen": 1949499392
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.266730861819934e-05,
+      "loss": 1.3441,
+      "theoretical_loss": 3.466178565514026,
+      "tokens_seen": 1949630464
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.26592842240411e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.4661591997325107,
+      "tokens_seen": 1949761536
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.265125982988285e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4661398356173008,
+      "tokens_seen": 1949892608
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.264323543572461e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4661204731681416,
+      "tokens_seen": 1950023680
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.263521104156637e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.466101112384778,
+      "tokens_seen": 1950154752
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.262718664740813e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.466081753266954,
+      "tokens_seen": 1950285824
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.261916225324989e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.466062395814415,
+      "tokens_seen": 1950416896
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.261113785909164e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.4660430400269058,
+      "tokens_seen": 1950547968
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.260311346493341e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.4660236859041706,
+      "tokens_seen": 1950679040
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.259508907077516e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.4660043334459556,
+      "tokens_seen": 1950810112
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.258706467661693e-05,
+      "loss": 1.2067,
+      "theoretical_loss": 3.465984982652005,
+      "tokens_seen": 1950941184
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.257904028245868e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.465965633522064,
+      "tokens_seen": 1951072256
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.257101588830045e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4659462860558774,
+      "tokens_seen": 1951203328
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.25629914941422e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.465926940253191,
+      "tokens_seen": 1951334400
+    },
+    {
+      "epoch": 0.18,
+      "objective/train/docs_used": 656717,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1874163150787354,
+      "objective/train/theoretical_loss": 3.46590759611375,
+      "objective/train/tokens_used": 321924576,
+      "theoretical_loss": 3.46590759611375,
+      "tokens_seen": 1951465472
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.255496709998395e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.46590759611375,
+      "tokens_seen": 1951465472
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.254694270582572e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4658882536372992,
+      "tokens_seen": 1951596544
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.253891831166747e-05,
+      "loss": 1.2931,
+      "theoretical_loss": 3.4658689128235842,
+      "tokens_seen": 1951727616
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.253089391750924e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.465849573672351,
+      "tokens_seen": 1951858688
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.252286952335099e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.465830236183344,
+      "tokens_seen": 1951989760
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.251484512919275e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.4658109003563093,
+      "tokens_seen": 1952120832
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.250682073503451e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.465791566190993,
+      "tokens_seen": 1952251904
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.249879634087627e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4657722336871397,
+      "tokens_seen": 1952382976
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.249077194671803e-05,
+      "loss": 1.3931,
+      "theoretical_loss": 3.4657529028444958,
+      "tokens_seen": 1952514048
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.248274755255979e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.465733573662807,
+      "tokens_seen": 1952645120
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.247472315840154e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.465714246141819,
+      "tokens_seen": 1952776192
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.24666987642433e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.465694920281277,
+      "tokens_seen": 1952907264
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.245867437008506e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4656755960809282,
+      "tokens_seen": 1953038336
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.245064997592683e-05,
+      "loss": 1.4007,
+      "theoretical_loss": 3.465656273540518,
+      "tokens_seen": 1953169408
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.244262558176858e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.465636952659792,
+      "tokens_seen": 1953300480
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.243460118761035e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4656176334384967,
+      "tokens_seen": 1953431552
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.24265767934521e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4655983158763783,
+      "tokens_seen": 1953562624
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.241855239929385e-05,
+      "loss": 1.3308,
+      "theoretical_loss": 3.465578999973183,
+      "tokens_seen": 1953693696
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.241052800513562e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.465559685728657,
+      "tokens_seen": 1953824768
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.240250361097737e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.465540373142546,
+      "tokens_seen": 1953955840
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.239447921681914e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.4655210622145978,
+      "tokens_seen": 1954086912
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.238645482266089e-05,
+      "loss": 1.3417,
+      "theoretical_loss": 3.465501752944557,
+      "tokens_seen": 1954217984
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.237843042850264e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.4654824453321718,
+      "tokens_seen": 1954349056
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.237040603434441e-05,
+      "loss": 1.2755,
+      "theoretical_loss": 3.4654631393771878,
+      "tokens_seen": 1954480128
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.236238164018616e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.465443835079351,
+      "tokens_seen": 1954611200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.235435724602793e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.46542453243841,
+      "tokens_seen": 1954742272
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.234633285186968e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.4654052314541097,
+      "tokens_seen": 1954873344
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.233830845771145e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.4653859321261975,
+      "tokens_seen": 1955004416
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.23302840635532e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.46536663445442,
+      "tokens_seen": 1955135488
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.232225966939496e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.4653473384385247,
+      "tokens_seen": 1955266560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.231423527523672e-05,
+      "loss": 1.1765,
+      "theoretical_loss": 3.465328044078258,
+      "tokens_seen": 1955397632
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.230621088107848e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4653087513733665,
+      "tokens_seen": 1955528704
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.229818648692024e-05,
+      "loss": 1.1374,
+      "theoretical_loss": 3.465289460323598,
+      "tokens_seen": 1955659776
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.2290162092762e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.465270170928699,
+      "tokens_seen": 1955790848
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.228213769860375e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.4652508831884177,
+      "tokens_seen": 1955921920
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.227411330444552e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.4652315971025,
+      "tokens_seen": 1956052992
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.226608891028727e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.4652123126706935,
+      "tokens_seen": 1956184064
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.225806451612904e-05,
+      "loss": 1.2938,
+      "theoretical_loss": 3.4651930298927462,
+      "tokens_seen": 1956315136
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.225004012197079e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.4651737487684047,
+      "tokens_seen": 1956446208
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.224201572781256e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.4651544692974166,
+      "tokens_seen": 1956577280
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.223399133365431e-05,
+      "loss": 1.3088,
+      "theoretical_loss": 3.4651351914795296,
+      "tokens_seen": 1956708352
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.222596693949606e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.465115915314491,
+      "tokens_seen": 1956839424
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.221794254533783e-05,
+      "loss": 1.3503,
+      "theoretical_loss": 3.4650966408020487,
+      "tokens_seen": 1956970496
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.220991815117958e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.46507736794195,
+      "tokens_seen": 1957101568
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.220189375702135e-05,
+      "loss": 1.3446,
+      "theoretical_loss": 3.4650580967339426,
+      "tokens_seen": 1957232640
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.21938693628631e-05,
+      "loss": 1.165,
+      "theoretical_loss": 3.465038827177775,
+      "tokens_seen": 1957363712
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.218584496870485e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.465019559273194,
+      "tokens_seen": 1957494784
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.217782057454662e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.4650002930199477,
+      "tokens_seen": 1957625856
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.216979618038837e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.4649810284177844,
+      "tokens_seen": 1957756928
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.216177178623014e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.4649617654664526,
+      "tokens_seen": 1957888000
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 658749,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.213796615600586,
+      "objective/train/theoretical_loss": 3.4649425041656987,
+      "objective/train/tokens_used": 328478176,
+      "theoretical_loss": 3.4649425041656987,
+      "tokens_seen": 1958019072
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.21537473920719e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4649425041656987,
+      "tokens_seen": 1958019072
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.214572299791366e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.464923244515272,
+      "tokens_seen": 1958150144
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.213769860375541e-05,
+      "loss": 1.3104,
+      "theoretical_loss": 3.4649039865149205,
+      "tokens_seen": 1958281216
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.212967420959717e-05,
+      "loss": 1.3457,
+      "theoretical_loss": 3.4648847301643926,
+      "tokens_seen": 1958412288
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.212164981543893e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.4648654754634354,
+      "tokens_seen": 1958543360
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.211362542128069e-05,
+      "loss": 1.297,
+      "theoretical_loss": 3.464846222411799,
+      "tokens_seen": 1958674432
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.210560102712245e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.46482697100923,
+      "tokens_seen": 1958805504
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.209757663296421e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.464807721255478,
+      "tokens_seen": 1958936576
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.208955223880597e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.464788473150291,
+      "tokens_seen": 1959067648
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.208152784464773e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4647692266934174,
+      "tokens_seen": 1959198720
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.20735034504895e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.464749981884606,
+      "tokens_seen": 1959329792
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.206547905633125e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.464730738723606,
+      "tokens_seen": 1959460864
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.2057454662173e-05,
+      "loss": 1.2784,
+      "theoretical_loss": 3.464711497210165,
+      "tokens_seen": 1959591936
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.204943026801477e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4646922573440326,
+      "tokens_seen": 1959723008
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.204140587385652e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.4646730191249575,
+      "tokens_seen": 1959854080
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.203338147969829e-05,
+      "loss": 1.288,
+      "theoretical_loss": 3.4646537825526877,
+      "tokens_seen": 1959985152
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.202535708554004e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.4646345476269733,
+      "tokens_seen": 1960116224
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.20173326913818e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.4646153143475633,
+      "tokens_seen": 1960247296
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.200930829722356e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.4645960827142055,
+      "tokens_seen": 1960378368
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.200128390306531e-05,
+      "loss": 1.3329,
+      "theoretical_loss": 3.46457685272665,
+      "tokens_seen": 1960509440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.199325950890708e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4645576243846454,
+      "tokens_seen": 1960640512
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.198523511474883e-05,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4645383976879414,
+      "tokens_seen": 1960771584
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.19772107205906e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.4645191726362867,
+      "tokens_seen": 1960902656
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.196918632643235e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.464499949229431,
+      "tokens_seen": 1961033728
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.196116193227412e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.464480727467123,
+      "tokens_seen": 1961164800
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.195313753811587e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.464461507349113,
+      "tokens_seen": 1961295872
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.194511314395762e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4644422888751505,
+      "tokens_seen": 1961426944
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.193708874979939e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4644230720449842,
+      "tokens_seen": 1961558016
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.192906435564114e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.4644038568583646,
+      "tokens_seen": 1961689088
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.192103996148291e-05,
+      "loss": 1.2795,
+      "theoretical_loss": 3.4643846433150403,
+      "tokens_seen": 1961820160
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.191301556732466e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4643654314147616,
+      "tokens_seen": 1961951232
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.190499117316643e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.464346221157278,
+      "tokens_seen": 1962082304
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.189696677900818e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.46432701254234,
+      "tokens_seen": 1962213376
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.188894238484994e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.464307805569697,
+      "tokens_seen": 1962344448
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.18809179906917e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.4642886002390982,
+      "tokens_seen": 1962475520
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.187289359653346e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.4642693965502946,
+      "tokens_seen": 1962606592
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.186486920237522e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4642501945030357,
+      "tokens_seen": 1962737664
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.185684480821698e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.4642309940970715,
+      "tokens_seen": 1962868736
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.184882041405874e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.4642117953321523,
+      "tokens_seen": 1962999808
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.18407960199005e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.4641925982080286,
+      "tokens_seen": 1963130880
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.183277162574225e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.46417340272445,
+      "tokens_seen": 1963261952
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.182474723158402e-05,
+      "loss": 1.3216,
+      "theoretical_loss": 3.464154208881167,
+      "tokens_seen": 1963393024
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.181672283742577e-05,
+      "loss": 1.269,
+      "theoretical_loss": 3.464135016677931,
+      "tokens_seen": 1963524096
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.180869844326754e-05,
+      "loss": 1.3192,
+      "theoretical_loss": 3.46411582611449,
+      "tokens_seen": 1963655168
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.180067404910929e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.464096637190597,
+      "tokens_seen": 1963786240
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.179264965495106e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.464077449906001,
+      "tokens_seen": 1963917312
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.178462526079281e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.464058264260453,
+      "tokens_seen": 1964048384
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.177660086663458e-05,
+      "loss": 1.3428,
+      "theoretical_loss": 3.4640390802537038,
+      "tokens_seen": 1964179456
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.176857647247633e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.4640198978855032,
+      "tokens_seen": 1964310528
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.176055207831808e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4640007171556033,
+      "tokens_seen": 1964441600
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 660816,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3085967302322388,
+      "objective/train/theoretical_loss": 3.463981538063754,
+      "objective/train/tokens_used": 335031776,
+      "theoretical_loss": 3.463981538063754,
+      "tokens_seen": 1964572672
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.175252768415985e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.463981538063754,
+      "tokens_seen": 1964572672
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.17445032900016e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.463962360609706,
+      "tokens_seen": 1964703744
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.173647889584337e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.463943184793211,
+      "tokens_seen": 1964834816
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.172845450168512e-05,
+      "loss": 1.3422,
+      "theoretical_loss": 3.463924010614019,
+      "tokens_seen": 1964965888
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.172043010752689e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.4639048380718815,
+      "tokens_seen": 1965096960
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.171240571336864e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.46388566716655,
+      "tokens_seen": 1965228032
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.17043813192104e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4638664978977745,
+      "tokens_seen": 1965359104
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.169635692505216e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.463847330265307,
+      "tokens_seen": 1965490176
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.168833253089391e-05,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4638281642688984,
+      "tokens_seen": 1965621248
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.168030813673568e-05,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4638089999083004,
+      "tokens_seen": 1965752320
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.167228374257743e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.4637898371832634,
+      "tokens_seen": 1965883392
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.16642593484192e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4637706760935396,
+      "tokens_seen": 1966014464
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.165623495426095e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4637515166388804,
+      "tokens_seen": 1966145536
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.164821056010271e-05,
+      "loss": 1.1825,
+      "theoretical_loss": 3.463732358819037,
+      "tokens_seen": 1966276608
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.164018616594447e-05,
+      "loss": 1.2871,
+      "theoretical_loss": 3.4637132026337607,
+      "tokens_seen": 1966407680
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.163216177178623e-05,
+      "loss": 1.3018,
+      "theoretical_loss": 3.463694048082804,
+      "tokens_seen": 1966538752
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.1624137377628e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.4636748951659175,
+      "tokens_seen": 1966669824
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.161611298346975e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.463655743882854,
+      "tokens_seen": 1966800896
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.160808858931151e-05,
+      "loss": 1.2818,
+      "theoretical_loss": 3.463636594233364,
+      "tokens_seen": 1966931968
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.160006419515327e-05,
+      "loss": 1.3434,
+      "theoretical_loss": 3.4636174462172002,
+      "tokens_seen": 1967063040
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.159203980099502e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.463598299834114,
+      "tokens_seen": 1967194112
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.158401540683679e-05,
+      "loss": 1.3189,
+      "theoretical_loss": 3.463579155083858,
+      "tokens_seen": 1967325184
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.157599101267854e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.4635600119661833,
+      "tokens_seen": 1967456256
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.15679666185203e-05,
+      "loss": 1.2869,
+      "theoretical_loss": 3.463540870480843,
+      "tokens_seen": 1967587328
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.155994222436206e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.463521730627588,
+      "tokens_seen": 1967718400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.155191783020383e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.463502592406171,
+      "tokens_seen": 1967849472
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.154389343604558e-05,
+      "loss": 1.2994,
+      "theoretical_loss": 3.4634834558163443,
+      "tokens_seen": 1967980544
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.153586904188733e-05,
+      "loss": 1.2063,
+      "theoretical_loss": 3.46346432085786,
+      "tokens_seen": 1968111616
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.15278446477291e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.4634451875304704,
+      "tokens_seen": 1968242688
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.151982025357085e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.463426055833928,
+      "tokens_seen": 1968373760
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.151179585941262e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.463406925767985,
+      "tokens_seen": 1968504832
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.150377146525437e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.463387797332394,
+      "tokens_seen": 1968635904
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.149574707109614e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.4633686705269073,
+      "tokens_seen": 1968766976
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.148772267693789e-05,
+      "loss": 1.2818,
+      "theoretical_loss": 3.463349545351278,
+      "tokens_seen": 1968898048
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.147969828277966e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4633304218052583,
+      "tokens_seen": 1969029120
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.147167388862141e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.463311299888601,
+      "tokens_seen": 1969160192
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.146364949446317e-05,
+      "loss": 1.329,
+      "theoretical_loss": 3.4632921796010585,
+      "tokens_seen": 1969291264
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.145562510030493e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.463273060942384,
+      "tokens_seen": 1969422336
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.144760070614668e-05,
+      "loss": 1.1709,
+      "theoretical_loss": 3.4632539439123304,
+      "tokens_seen": 1969553408
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.143957631198845e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4632348285106502,
+      "tokens_seen": 1969684480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.14315519178302e-05,
+      "loss": 1.2669,
+      "theoretical_loss": 3.463215714737097,
+      "tokens_seen": 1969815552
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.142352752367197e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.463196602591423,
+      "tokens_seen": 1969946624
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.141550312951372e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4631774920733815,
+      "tokens_seen": 1970077696
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.140747873535548e-05,
+      "loss": 1.3939,
+      "theoretical_loss": 3.4631583831827264,
+      "tokens_seen": 1970208768
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.139945434119724e-05,
+      "loss": 1.2337,
+      "theoretical_loss": 3.4631392759192092,
+      "tokens_seen": 1970339840
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.1391429947039e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4631201702825853,
+      "tokens_seen": 1970470912
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.138340555288076e-05,
+      "loss": 1.3249,
+      "theoretical_loss": 3.463101066272606,
+      "tokens_seen": 1970601984
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.137538115872252e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.463081963889026,
+      "tokens_seen": 1970733056
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.136735676456428e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4630628631315976,
+      "tokens_seen": 1970864128
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.135933237040604e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.463043764000075,
+      "tokens_seen": 1970995200
+    },
+    {
+      "epoch": 0.19,
+      "objective/train/docs_used": 662847,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2778606414794922,
+      "objective/train/theoretical_loss": 3.4630246664942117,
+      "objective/train/tokens_used": 341585376,
+      "theoretical_loss": 3.4630246664942117,
+      "tokens_seen": 1971126272
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.135130797624779e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.4630246664942117,
+      "tokens_seen": 1971126272
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.134328358208956e-05,
+      "loss": 1.3501,
+      "theoretical_loss": 3.4630055706137606,
+      "tokens_seen": 1971257344
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.133525918793131e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.462986476358476,
+      "tokens_seen": 1971388416
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.132723479377308e-05,
+      "loss": 1.3031,
+      "theoretical_loss": 3.462967383728111,
+      "tokens_seen": 1971519488
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.131921039961483e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.4629482927224196,
+      "tokens_seen": 1971650560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.13111860054566e-05,
+      "loss": 1.3032,
+      "theoretical_loss": 3.4629292033411554,
+      "tokens_seen": 1971781632
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.130316161129835e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.4629101155840725,
+      "tokens_seen": 1971912704
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.12951372171401e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4628910294509243,
+      "tokens_seen": 1972043776
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.128711282298187e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4628719449414653,
+      "tokens_seen": 1972174848
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.127908842882362e-05,
+      "loss": 1.331,
+      "theoretical_loss": 3.462852862055449,
+      "tokens_seen": 1972305920
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.127106403466539e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.4628337807926295,
+      "tokens_seen": 1972436992
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.126303964050714e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.4628147011527615,
+      "tokens_seen": 1972568064
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.125501524634891e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.462795623135598,
+      "tokens_seen": 1972699136
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.124699085219066e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.462776546740894,
+      "tokens_seen": 1972830208
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.123896645803242e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.4627574719684038,
+      "tokens_seen": 1972961280
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.123094206387418e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.462738398817881,
+      "tokens_seen": 1973092352
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.122291766971594e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.4627193272890806,
+      "tokens_seen": 1973223424
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.12148932755577e-05,
+      "loss": 1.3566,
+      "theoretical_loss": 3.4627002573817567,
+      "tokens_seen": 1973354496
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.120686888139945e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4626811890956644,
+      "tokens_seen": 1973485568
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.119884448724122e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.4626621224305567,
+      "tokens_seen": 1973616640
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.119082009308297e-05,
+      "loss": 1.3031,
+      "theoretical_loss": 3.4626430573861895,
+      "tokens_seen": 1973747712
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.118279569892473e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.462623993962317,
+      "tokens_seen": 1973878784
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.11747713047665e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.462604932158694,
+      "tokens_seen": 1974009856
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.116674691060825e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.462585871975075,
+      "tokens_seen": 1974140928
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.115872251645001e-05,
+      "loss": 1.3046,
+      "theoretical_loss": 3.4625668134112146,
+      "tokens_seen": 1974272000
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.115069812229177e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4625477564668685,
+      "tokens_seen": 1974403072
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.114267372813353e-05,
+      "loss": 1.2817,
+      "theoretical_loss": 3.4625287011417907,
+      "tokens_seen": 1974534144
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.113464933397529e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.462509647435737,
+      "tokens_seen": 1974665216
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.112662493981705e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.462490595348461,
+      "tokens_seen": 1974796288
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.111860054565881e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.4624715448797185,
+      "tokens_seen": 1974927360
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.111057615150056e-05,
+      "loss": 1.3083,
+      "theoretical_loss": 3.4624524960292655,
+      "tokens_seen": 1975058432
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.110255175734233e-05,
+      "loss": 1.3741,
+      "theoretical_loss": 3.4624334487968556,
+      "tokens_seen": 1975189504
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.109452736318408e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.462414403182245,
+      "tokens_seen": 1975320576
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.108650296902585e-05,
+      "loss": 1.3212,
+      "theoretical_loss": 3.462395359185188,
+      "tokens_seen": 1975451648
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.10784785748676e-05,
+      "loss": 1.3245,
+      "theoretical_loss": 3.4623763168054413,
+      "tokens_seen": 1975582720
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.107045418070937e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.4623572760427592,
+      "tokens_seen": 1975713792
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.106242978655112e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.4623382368968976,
+      "tokens_seen": 1975844864
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.105440539239287e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.462319199367612,
+      "tokens_seen": 1975975936
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.104638099823464e-05,
+      "loss": 1.3196,
+      "theoretical_loss": 3.4623001634546577,
+      "tokens_seen": 1976107008
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.103835660407639e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.46228112915779,
+      "tokens_seen": 1976238080
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.103033220991816e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.462262096476765,
+      "tokens_seen": 1976369152
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.102230781575991e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.462243065411338,
+      "tokens_seen": 1976500224
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.101428342160168e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.462224035961265,
+      "tokens_seen": 1976631296
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.100625902744343e-05,
+      "loss": 1.3578,
+      "theoretical_loss": 3.4622050081263023,
+      "tokens_seen": 1976762368
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.099823463328519e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.462185981906205,
+      "tokens_seen": 1976893440
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.099021023912695e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.4621669573007288,
+      "tokens_seen": 1977024512
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.09821858449687e-05,
+      "loss": 1.3354,
+      "theoretical_loss": 3.4621479343096304,
+      "tokens_seen": 1977155584
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.097416145081047e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.462128912932665,
+      "tokens_seen": 1977286656
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.096613705665222e-05,
+      "loss": 1.3232,
+      "theoretical_loss": 3.462109893169589,
+      "tokens_seen": 1977417728
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.095811266249399e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.462090875020159,
+      "tokens_seen": 1977548800
+    },
+    {
+      "epoch": 0.2,
+      "objective/train/docs_used": 664930,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2078478336334229,
+      "objective/train/theoretical_loss": 3.462071858484131,
+      "objective/train/tokens_used": 348138976,
+      "theoretical_loss": 3.462071858484131,
+      "tokens_seen": 1977679872
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.095008826833574e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.462071858484131,
+      "tokens_seen": 1977679872
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.09420638741775e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.46205284356126,
+      "tokens_seen": 1977810944
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.093403948001926e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.462033830251304,
+      "tokens_seen": 1977942016
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.092601508586102e-05,
+      "loss": 1.342,
+      "theoretical_loss": 3.4620148185540183,
+      "tokens_seen": 1978073088
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.091799069170278e-05,
+      "loss": 1.2816,
+      "theoretical_loss": 3.461995808469159,
+      "tokens_seen": 1978204160
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.090996629754454e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.461976799996484,
+      "tokens_seen": 1978335232
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.09019419033863e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.461957793135748,
+      "tokens_seen": 1978466304
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.089391750922806e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4619387878867087,
+      "tokens_seen": 1978597376
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.088589311506981e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4619197842491225,
+      "tokens_seen": 1978728448
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.087786872091158e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.461900782222745,
+      "tokens_seen": 1978859520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.086984432675333e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.4618817818073344,
+      "tokens_seen": 1978990592
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.08618199325951e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.461862783002647,
+      "tokens_seen": 1979121664
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.085379553843685e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.461843785808439,
+      "tokens_seen": 1979252736
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.084577114427862e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.461824790224468,
+      "tokens_seen": 1979383808
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.083774675012037e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.46180579625049,
+      "tokens_seen": 1979514880
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.082972235596214e-05,
+      "loss": 1.3022,
+      "theoretical_loss": 3.461786803886263,
+      "tokens_seen": 1979645952
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.082169796180389e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.461767813131543,
+      "tokens_seen": 1979777024
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.081367356764564e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.4617488239860874,
+      "tokens_seen": 1979908096
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.080564917348741e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.4617298364496536,
+      "tokens_seen": 1980039168
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.079762477932916e-05,
+      "loss": 1.2702,
+      "theoretical_loss": 3.461710850521999,
+      "tokens_seen": 1980170240
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.078960038517093e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.46169186620288,
+      "tokens_seen": 1980301312
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.078157599101268e-05,
+      "loss": 1.352,
+      "theoretical_loss": 3.4616728834920547,
+      "tokens_seen": 1980432384
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.077355159685445e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.46165390238928,
+      "tokens_seen": 1980563456
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.07655272026962e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.461634922894313,
+      "tokens_seen": 1980694528
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.075750280853796e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.461615945006911,
+      "tokens_seen": 1980825600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.074947841437972e-05,
+      "loss": 1.3168,
+      "theoretical_loss": 3.4615969687268326,
+      "tokens_seen": 1980956672
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.074145402022148e-05,
+      "loss": 1.3319,
+      "theoretical_loss": 3.4615779940538345,
+      "tokens_seen": 1981087744
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.073342962606324e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.461559020987674,
+      "tokens_seen": 1981218816
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.0725405231905e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.46154004952811,
+      "tokens_seen": 1981349888
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.071738083774676e-05,
+      "loss": 1.3876,
+      "theoretical_loss": 3.4615210796748985,
+      "tokens_seen": 1981480960
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.070935644358851e-05,
+      "loss": 1.3208,
+      "theoretical_loss": 3.4615021114277984,
+      "tokens_seen": 1981612032
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.070133204943027e-05,
+      "loss": 1.317,
+      "theoretical_loss": 3.461483144786567,
+      "tokens_seen": 1981743104
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.069330765527203e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.4614641797509624,
+      "tokens_seen": 1981874176
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.068528326111379e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.4614452163207425,
+      "tokens_seen": 1982005248
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.067725886695555e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.4614262544956653,
+      "tokens_seen": 1982136320
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.066923447279731e-05,
+      "loss": 1.2927,
+      "theoretical_loss": 3.4614072942754883,
+      "tokens_seen": 1982267392
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.066121007863907e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.4613883356599704,
+      "tokens_seen": 1982398464
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.065318568448083e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.461369378648869,
+      "tokens_seen": 1982529536
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.064516129032258e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.4613504232419423,
+      "tokens_seen": 1982660608
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.063713689616435e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.461331469438949,
+      "tokens_seen": 1982791680
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.06291125020061e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.461312517239647,
+      "tokens_seen": 1982922752
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.062108810784787e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4612935666437945,
+      "tokens_seen": 1983053824
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.061306371368962e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.46127461765115,
+      "tokens_seen": 1983184896
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.060503931953139e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.4612556702614725,
+      "tokens_seen": 1983315968
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.059701492537314e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.46123672447452,
+      "tokens_seen": 1983447040
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.058899053121489e-05,
+      "loss": 1.2961,
+      "theoretical_loss": 3.461217780290051,
+      "tokens_seen": 1983578112
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.058096613705666e-05,
+      "loss": 1.1509,
+      "theoretical_loss": 3.4611988377078236,
+      "tokens_seen": 1983709184
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.057294174289841e-05,
+      "loss": 1.1785,
+      "theoretical_loss": 3.4611798967275975,
+      "tokens_seen": 1983840256
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.056491734874018e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.46116095734913,
+      "tokens_seen": 1983971328
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.055689295458193e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.461142019572182,
+      "tokens_seen": 1984102400
+    },
+    {
+      "epoch": 0.2,
+      "objective/train/docs_used": 666996,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2686578035354614,
+      "objective/train/theoretical_loss": 3.46112308339651,
+      "objective/train/tokens_used": 354692576,
+      "theoretical_loss": 3.46112308339651,
+      "tokens_seen": 1984233472
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.05488685604237e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.46112308339651,
+      "tokens_seen": 1984233472
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.054084416626545e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.4611041488218737,
+      "tokens_seen": 1984364544
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.053281977210722e-05,
+      "loss": 1.1994,
+      "theoretical_loss": 3.461085215848032,
+      "tokens_seen": 1984495616
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.052479537794897e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.4610662844747444,
+      "tokens_seen": 1984626688
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.051677098379073e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.461047354701769,
+      "tokens_seen": 1984757760
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.050874658963249e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.4610284265288653,
+      "tokens_seen": 1984888832
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.050072219547425e-05,
+      "loss": 1.3131,
+      "theoretical_loss": 3.461009499955793,
+      "tokens_seen": 1985019904
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.049269780131601e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4609905749823104,
+      "tokens_seen": 1985150976
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.048467340715777e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.4609716516081765,
+      "tokens_seen": 1985282048
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.047664901299953e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.460952729833151,
+      "tokens_seen": 1985413120
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.046862461884128e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.4609338096569937,
+      "tokens_seen": 1985544192
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.046060022468304e-05,
+      "loss": 1.3301,
+      "theoretical_loss": 3.4609148910794634,
+      "tokens_seen": 1985675264
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.04525758305248e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4608959741003193,
+      "tokens_seen": 1985806336
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.044455143636656e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.460877058719322,
+      "tokens_seen": 1985937408
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.043652704220832e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.4608581449362292,
+      "tokens_seen": 1986068480
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.042850264805008e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.4608392327508017,
+      "tokens_seen": 1986199552
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.042047825389184e-05,
+      "loss": 1.3544,
+      "theoretical_loss": 3.4608203221627987,
+      "tokens_seen": 1986330624
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.04124538597336e-05,
+      "loss": 1.2822,
+      "theoretical_loss": 3.4608014131719806,
+      "tokens_seen": 1986461696
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.040442946557535e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.4607825057781065,
+      "tokens_seen": 1986592768
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.039640507141712e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.460763599980936,
+      "tokens_seen": 1986723840
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.038838067725887e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.460744695780229,
+      "tokens_seen": 1986854912
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.038035628310064e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.4607257931757465,
+      "tokens_seen": 1986985984
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.037233188894239e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.460706892167247,
+      "tokens_seen": 1987117056
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.036430749478416e-05,
+      "loss": 1.2934,
+      "theoretical_loss": 3.4606879927544907,
+      "tokens_seen": 1987248128
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.035628310062591e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.460669094937238,
+      "tokens_seen": 1987379200
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.034825870646766e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4606501987152494,
+      "tokens_seen": 1987510272
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.034023431230943e-05,
+      "loss": 1.3342,
+      "theoretical_loss": 3.4606313040882846,
+      "tokens_seen": 1987641344
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.033220991815118e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.460612411056103,
+      "tokens_seen": 1987772416
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.032418552399295e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.4605935196184667,
+      "tokens_seen": 1987903488
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.03161611298347e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.4605746297751345,
+      "tokens_seen": 1988034560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 8.030813673567647e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.4605557415258668,
+      "tokens_seen": 1988165632
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.030011234151822e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.460536854870425,
+      "tokens_seen": 1988296704
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.029208794735998e-05,
+      "loss": 1.2586,
+      "theoretical_loss": 3.460517969808569,
+      "tokens_seen": 1988427776
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.028406355320174e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.4604990863400586,
+      "tokens_seen": 1988558848
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.02760391590435e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.4604802044646554,
+      "tokens_seen": 1988689920
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.026801476488526e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.46046132418212,
+      "tokens_seen": 1988820992
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.025999037072702e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.460442445492212,
+      "tokens_seen": 1988952064
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.025196597656878e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.460423568394693,
+      "tokens_seen": 1989083136
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.024394158241054e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.4604046928893237,
+      "tokens_seen": 1989214208
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.023591718825229e-05,
+      "loss": 1.204,
+      "theoretical_loss": 3.4603858189758645,
+      "tokens_seen": 1989345280
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.022789279409406e-05,
+      "loss": 1.1703,
+      "theoretical_loss": 3.4603669466540774,
+      "tokens_seen": 1989476352
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.021986839993581e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4603480759237217,
+      "tokens_seen": 1989607424
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.021184400577757e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.4603292067845595,
+      "tokens_seen": 1989738496
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.020381961161933e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.4603103392363512,
+      "tokens_seen": 1989869568
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.01957952174611e-05,
+      "loss": 1.3308,
+      "theoretical_loss": 3.4602914732788586,
+      "tokens_seen": 1990000640
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.018777082330285e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.4602726089118416,
+      "tokens_seen": 1990131712
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.017974642914461e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.460253746135063,
+      "tokens_seen": 1990262784
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.017172203498637e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.4602348849482825,
+      "tokens_seen": 1990393856
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.016369764082812e-05,
+      "loss": 1.3525,
+      "theoretical_loss": 3.460216025351263,
+      "tokens_seen": 1990524928
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.015567324666989e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.4601971673437637,
+      "tokens_seen": 1990656000
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 669188,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3270471096038818,
+      "objective/train/theoretical_loss": 3.4601783109255484,
+      "objective/train/tokens_used": 361246176,
+      "theoretical_loss": 3.4601783109255484,
+      "tokens_seen": 1990787072
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.014764885251164e-05,
+      "loss": 1.2887,
+      "theoretical_loss": 3.4601783109255484,
+      "tokens_seen": 1990787072
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.013962445835341e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.4601594560963767,
+      "tokens_seen": 1990918144
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.013160006419516e-05,
+      "loss": 1.3376,
+      "theoretical_loss": 3.4601406028560104,
+      "tokens_seen": 1991049216
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.012357567003693e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.4601217512042117,
+      "tokens_seen": 1991180288
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.011555127587868e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.4601029011407425,
+      "tokens_seen": 1991311360
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.010752688172043e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.460084052665363,
+      "tokens_seen": 1991442432
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.00995024875622e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.460065205777836,
+      "tokens_seen": 1991573504
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.009147809340395e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.460046360477923,
+      "tokens_seen": 1991704576
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.008345369924572e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.4600275167653853,
+      "tokens_seen": 1991835648
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.007542930508747e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4600086746399854,
+      "tokens_seen": 1991966720
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.006740491092924e-05,
+      "loss": 1.295,
+      "theoretical_loss": 3.459989834101485,
+      "tokens_seen": 1992097792
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.005938051677099e-05,
+      "loss": 1.3093,
+      "theoretical_loss": 3.4599709951496465,
+      "tokens_seen": 1992228864
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.005135612261275e-05,
+      "loss": 1.1812,
+      "theoretical_loss": 3.4599521577842305,
+      "tokens_seen": 1992359936
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.004333172845451e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.459933322005001,
+      "tokens_seen": 1992491008
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.003530733429627e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4599144878117185,
+      "tokens_seen": 1992622080
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.002728294013803e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.4598956552041455,
+      "tokens_seen": 1992753152
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.001925854597979e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.4598768241820452,
+      "tokens_seen": 1992884224
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.001123415182155e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.4598579947451786,
+      "tokens_seen": 1993015296
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 8.00032097576633e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4598391668933086,
+      "tokens_seen": 1993146368
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.999518536350506e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.459820340626198,
+      "tokens_seen": 1993277440
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.998716096934683e-05,
+      "loss": 1.2823,
+      "theoretical_loss": 3.459801515943608,
+      "tokens_seen": 1993408512
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.997913657518858e-05,
+      "loss": 1.3456,
+      "theoretical_loss": 3.4597826928453017,
+      "tokens_seen": 1993539584
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.997111218103034e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.4597638713310426,
+      "tokens_seen": 1993670656
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.99630877868721e-05,
+      "loss": 1.3544,
+      "theoretical_loss": 3.4597450514005916,
+      "tokens_seen": 1993801728
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.995506339271386e-05,
+      "loss": 1.2169,
+      "theoretical_loss": 3.459726233053712,
+      "tokens_seen": 1993932800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.994703899855562e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.459707416290167,
+      "tokens_seen": 1994063872
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.993901460439737e-05,
+      "loss": 1.2124,
+      "theoretical_loss": 3.459688601109719,
+      "tokens_seen": 1994194944
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.993099021023914e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.45966978751213,
+      "tokens_seen": 1994326016
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.992296581608089e-05,
+      "loss": 1.3241,
+      "theoretical_loss": 3.459650975497164,
+      "tokens_seen": 1994457088
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.991494142192266e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.4596321650645834,
+      "tokens_seen": 1994588160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.990691702776441e-05,
+      "loss": 1.2994,
+      "theoretical_loss": 3.4596133562141507,
+      "tokens_seen": 1994719232
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.989889263360616e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.4595945489456295,
+      "tokens_seen": 1994850304
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.989086823944793e-05,
+      "loss": 1.3148,
+      "theoretical_loss": 3.459575743258782,
+      "tokens_seen": 1994981376
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.988284384528968e-05,
+      "loss": 1.3726,
+      "theoretical_loss": 3.4595569391533725,
+      "tokens_seen": 1995112448
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.987481945113145e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.4595381366291633,
+      "tokens_seen": 1995243520
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.98667950569732e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4595193356859175,
+      "tokens_seen": 1995374592
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.985877066281497e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.4595005363233984,
+      "tokens_seen": 1995505664
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.985074626865672e-05,
+      "loss": 1.3011,
+      "theoretical_loss": 3.45948173854137,
+      "tokens_seen": 1995636736
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.984272187449848e-05,
+      "loss": 1.2488,
+      "theoretical_loss": 3.459462942339595,
+      "tokens_seen": 1995767808
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.983469748034024e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4594441477178366,
+      "tokens_seen": 1995898880
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.9826673086182e-05,
+      "loss": 1.1919,
+      "theoretical_loss": 3.459425354675859,
+      "tokens_seen": 1996029952
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.981864869202376e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.4594065632134248,
+      "tokens_seen": 1996161024
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.981062429786552e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.459387773330298,
+      "tokens_seen": 1996292096
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.980259990370727e-05,
+      "loss": 1.1885,
+      "theoretical_loss": 3.4593689850262423,
+      "tokens_seen": 1996423168
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.979457550954904e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.459350198301021,
+      "tokens_seen": 1996554240
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.978655111539079e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.4593314131543984,
+      "tokens_seen": 1996685312
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.977852672123256e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.459312629586137,
+      "tokens_seen": 1996816384
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.977050232707431e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.459293847596002,
+      "tokens_seen": 1996947456
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.976247793291608e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.459275067183756,
+      "tokens_seen": 1997078528
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.975445353875783e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4592562883491644,
+      "tokens_seen": 1997209600
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 671479,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2921390533447266,
+      "objective/train/theoretical_loss": 3.45923751109199,
+      "objective/train/tokens_used": 367799776,
+      "theoretical_loss": 3.45923751109199,
+      "tokens_seen": 1997340672
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.974642914459958e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.45923751109199,
+      "tokens_seen": 1997340672
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.973840475044135e-05,
+      "loss": 1.2653,
+      "theoretical_loss": 3.459218735411997,
+      "tokens_seen": 1997471744
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.97303803562831e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.4591999613089492,
+      "tokens_seen": 1997602816
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.972235596212487e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.4591811887826114,
+      "tokens_seen": 1997733888
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.971433156796662e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.459162417832747,
+      "tokens_seen": 1997864960
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.970630717380837e-05,
+      "loss": 1.217,
+      "theoretical_loss": 3.4591436484591207,
+      "tokens_seen": 1997996032
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.969828277965014e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4591248806614967,
+      "tokens_seen": 1998127104
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.96902583854919e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.4591061144396393,
+      "tokens_seen": 1998258176
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.968223399133366e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.459087349793313,
+      "tokens_seen": 1998389248
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.967420959717541e-05,
+      "loss": 1.1549,
+      "theoretical_loss": 3.4590685867222812,
+      "tokens_seen": 1998520320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.966618520301718e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.4590498252263098,
+      "tokens_seen": 1998651392
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.965816080885893e-05,
+      "loss": 1.1673,
+      "theoretical_loss": 3.4590310653051617,
+      "tokens_seen": 1998782464
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.965013641470069e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.459012306958603,
+      "tokens_seen": 1998913536
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.964211202054245e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4589935501863978,
+      "tokens_seen": 1999044608
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.96340876263842e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4589747949883103,
+      "tokens_seen": 1999175680
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.962606323222597e-05,
+      "loss": 1.2832,
+      "theoretical_loss": 3.4589560413641056,
+      "tokens_seen": 1999306752
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.961803883806773e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.458937289313549,
+      "tokens_seen": 1999437824
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.961001444390948e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.4589185388364037,
+      "tokens_seen": 1999568896
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.960199004975125e-05,
+      "loss": 1.3033,
+      "theoretical_loss": 3.458899789932436,
+      "tokens_seen": 1999699968
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.9593965655593e-05,
+      "loss": 1.3061,
+      "theoretical_loss": 3.4588810426014103,
+      "tokens_seen": 1999831040
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.958594126143477e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.4588622968430918,
+      "tokens_seen": 1999962112
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.957791686727652e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.458843552657245,
+      "tokens_seen": 2000093184
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.956989247311829e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.4588248100436356,
+      "tokens_seen": 2000224256
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.956186807896004e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.458806069002028,
+      "tokens_seen": 2000355328
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.955384368480179e-05,
+      "loss": 1.3327,
+      "theoretical_loss": 3.4587873295321883,
+      "tokens_seen": 2000486400
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.954581929064356e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4587685916338806,
+      "tokens_seen": 2000617472
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.953779489648531e-05,
+      "loss": 1.3132,
+      "theoretical_loss": 3.458749855306871,
+      "tokens_seen": 2000748544
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.952977050232708e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.4587311205509246,
+      "tokens_seen": 2000879616
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.952174610816883e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4587123873658063,
+      "tokens_seen": 2001010688
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.951372171401059e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.458693655751283,
+      "tokens_seen": 2001141760
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.950569731985235e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.4586749257071174,
+      "tokens_seen": 2001272832
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.94976729256941e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.458656197233078,
+      "tokens_seen": 2001403904
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.948964853153587e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.458637470328928,
+      "tokens_seen": 2001534976
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.948162413737762e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.458618744994435,
+      "tokens_seen": 2001666048
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.947359974321939e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.4586000212293633,
+      "tokens_seen": 2001797120
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.946557534906114e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.458581299033479,
+      "tokens_seen": 2001928192
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.94575509549029e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.458562578406548,
+      "tokens_seen": 2002059264
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.944952656074466e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.458543859348336,
+      "tokens_seen": 2002190336
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.944150216658642e-05,
+      "loss": 1.3164,
+      "theoretical_loss": 3.4585251418586087,
+      "tokens_seen": 2002321408
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.943347777242818e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.4585064259371325,
+      "tokens_seen": 2002452480
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.942545337826994e-05,
+      "loss": 1.2933,
+      "theoretical_loss": 3.4584877115836727,
+      "tokens_seen": 2002583552
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.94174289841117e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.4584689987979957,
+      "tokens_seen": 2002714624
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.940940458995346e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.4584502875798675,
+      "tokens_seen": 2002845696
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.940138019579521e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.458431577929055,
+      "tokens_seen": 2002976768
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.939335580163698e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.458412869845323,
+      "tokens_seen": 2003107840
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.938533140747873e-05,
+      "loss": 1.2136,
+      "theoretical_loss": 3.458394163328438,
+      "tokens_seen": 2003238912
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.93773070133205e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.458375458378167,
+      "tokens_seen": 2003369984
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.936928261916225e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.458356754994276,
+      "tokens_seen": 2003501056
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.936125822500402e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.458338053176531,
+      "tokens_seen": 2003632128
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.935323383084577e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.458319352924699,
+      "tokens_seen": 2003763200
+    },
+    {
+      "epoch": 0.21,
+      "objective/train/docs_used": 673566,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.365978717803955,
+      "objective/train/theoretical_loss": 3.4583006542385464,
+      "objective/train/tokens_used": 374353376,
+      "theoretical_loss": 3.4583006542385464,
+      "tokens_seen": 2003894272
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.934520943668752e-05,
+      "loss": 1.3017,
+      "theoretical_loss": 3.4583006542385464,
+      "tokens_seen": 2003894272
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.933718504252929e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.4582819571178387,
+      "tokens_seen": 2004025344
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.932916064837104e-05,
+      "loss": 1.2877,
+      "theoretical_loss": 3.458263261562344,
+      "tokens_seen": 2004156416
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.932113625421281e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.458244567571828,
+      "tokens_seen": 2004287488
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.931311186005456e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4582258751460575,
+      "tokens_seen": 2004418560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.930508746589633e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.458207184284799,
+      "tokens_seen": 2004549632
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.929706307173808e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.4581884949878194,
+      "tokens_seen": 2004680704
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.928903867757984e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.458169807254886,
+      "tokens_seen": 2004811776
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.92810142834216e-05,
+      "loss": 1.2871,
+      "theoretical_loss": 3.4581511210857654,
+      "tokens_seen": 2004942848
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.927298988926336e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.4581324364802244,
+      "tokens_seen": 2005073920
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.926496549510512e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.45811375343803,
+      "tokens_seen": 2005204992
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.925694110094687e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.458095071958949,
+      "tokens_seen": 2005336064
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.924891670678864e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.4580763920427486,
+      "tokens_seen": 2005467136
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.92408923126304e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.458057713689196,
+      "tokens_seen": 2005598208
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.923286791847216e-05,
+      "loss": 1.2656,
+      "theoretical_loss": 3.458039036898059,
+      "tokens_seen": 2005729280
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.922484352431391e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.4580203616691034,
+      "tokens_seen": 2005860352
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.921681913015567e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.4580016880020974,
+      "tokens_seen": 2005991424
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.920879473599743e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.457983015896809,
+      "tokens_seen": 2006122496
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.920077034183919e-05,
+      "loss": 1.222,
+      "theoretical_loss": 3.4579643453530036,
+      "tokens_seen": 2006253568
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.919274594768095e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.4579456763704504,
+      "tokens_seen": 2006384640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.918472155352271e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.457927008948916,
+      "tokens_seen": 2006515712
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.917669715936447e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.4579083430881683,
+      "tokens_seen": 2006646784
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.916867276520623e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.457889678787975,
+      "tokens_seen": 2006777856
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.916064837104798e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.4578710160481023,
+      "tokens_seen": 2006908928
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.915262397688975e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.45785235486832,
+      "tokens_seen": 2007040000
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.91445995827315e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.457833695248395,
+      "tokens_seen": 2007171072
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.913657518857327e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.4578150371880936,
+      "tokens_seen": 2007302144
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.912855079441502e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.457796380687186,
+      "tokens_seen": 2007433216
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.912052640025679e-05,
+      "loss": 1.3319,
+      "theoretical_loss": 3.4577777257454385,
+      "tokens_seen": 2007564288
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.911250200609854e-05,
+      "loss": 1.3217,
+      "theoretical_loss": 3.4577590723626193,
+      "tokens_seen": 2007695360
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.910447761194029e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4577404205384967,
+      "tokens_seen": 2007826432
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.909645321778206e-05,
+      "loss": 1.3115,
+      "theoretical_loss": 3.4577217702728387,
+      "tokens_seen": 2007957504
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.908842882362381e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.4577031215654133,
+      "tokens_seen": 2008088576
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.908040442946558e-05,
+      "loss": 1.3317,
+      "theoretical_loss": 3.4576844744159883,
+      "tokens_seen": 2008219648
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.907238003530733e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.4576658288243323,
+      "tokens_seen": 2008350720
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.90643556411491e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.457647184790213,
+      "tokens_seen": 2008481792
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.905633124699085e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.4576285423133992,
+      "tokens_seen": 2008612864
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.90483068528326e-05,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4576099013936585,
+      "tokens_seen": 2008743936
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.904028245867437e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.45759126203076,
+      "tokens_seen": 2008875008
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.903225806451613e-05,
+      "loss": 1.1775,
+      "theoretical_loss": 3.457572624224472,
+      "tokens_seen": 2009006080
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.902423367035789e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.4575539879745625,
+      "tokens_seen": 2009137152
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.901620927619964e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.4575353532808,
+      "tokens_seen": 2009268224
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.900818488204141e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.457516720142954,
+      "tokens_seen": 2009399296
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.900016048788316e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.4574980885607927,
+      "tokens_seen": 2009530368
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.899213609372492e-05,
+      "loss": 1.2828,
+      "theoretical_loss": 3.457479458534084,
+      "tokens_seen": 2009661440
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.898411169956668e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.457460830062597,
+      "tokens_seen": 2009792512
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.897608730540844e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.457442203146101,
+      "tokens_seen": 2009923584
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.89680629112502e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4574235777843643,
+      "tokens_seen": 2010054656
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.896003851709196e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.457404953977156,
+      "tokens_seen": 2010185728
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.895201412293372e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.4573863317242446,
+      "tokens_seen": 2010316800
+    },
+    {
+      "epoch": 0.22,
+      "objective/train/docs_used": 675820,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3118029832839966,
+      "objective/train/theoretical_loss": 3.457367711025399,
+      "objective/train/tokens_used": 380906976,
+      "theoretical_loss": 3.457367711025399,
+      "tokens_seen": 2010447872
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.894398972877548e-05,
+      "loss": 1.2911,
+      "theoretical_loss": 3.457367711025399,
+      "tokens_seen": 2010447872
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.893596533461724e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4573490918803893,
+      "tokens_seen": 2010578944
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.8927940940459e-05,
+      "loss": 1.3442,
+      "theoretical_loss": 3.4573304742889834,
+      "tokens_seen": 2010710016
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.891991654630075e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.457311858250951,
+      "tokens_seen": 2010841088
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.891189215214252e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.4572932437660606,
+      "tokens_seen": 2010972160
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.890386775798427e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.457274630834082,
+      "tokens_seen": 2011103232
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.889584336382604e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4572560194547846,
+      "tokens_seen": 2011234304
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.888781896966779e-05,
+      "loss": 1.3149,
+      "theoretical_loss": 3.457237409627937,
+      "tokens_seen": 2011365376
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.887979457550956e-05,
+      "loss": 1.3123,
+      "theoretical_loss": 3.457218801353309,
+      "tokens_seen": 2011496448
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.887177018135131e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.45720019463067,
+      "tokens_seen": 2011627520
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.886374578719306e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.4571815894597897,
+      "tokens_seen": 2011758592
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.885572139303483e-05,
+      "loss": 1.3447,
+      "theoretical_loss": 3.457162985840437,
+      "tokens_seen": 2011889664
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.884769699887658e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4571443837723814,
+      "tokens_seen": 2012020736
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.883967260471835e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.4571257832553934,
+      "tokens_seen": 2012151808
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.88316482105601e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.457107184289242,
+      "tokens_seen": 2012282880
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.882362381640187e-05,
+      "loss": 1.3198,
+      "theoretical_loss": 3.457088586873697,
+      "tokens_seen": 2012413952
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.881559942224362e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.4570699910085274,
+      "tokens_seen": 2012545024
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.880757502808538e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4570513966935046,
+      "tokens_seen": 2012676096
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.879955063392714e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.457032803928397,
+      "tokens_seen": 2012807168
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.87915262397689e-05,
+      "loss": 1.1774,
+      "theoretical_loss": 3.4570142127129753,
+      "tokens_seen": 2012938240
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.878350184561066e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4569956230470087,
+      "tokens_seen": 2013069312
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.877547745145242e-05,
+      "loss": 1.3015,
+      "theoretical_loss": 3.4569770349302686,
+      "tokens_seen": 2013200384
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.876745305729418e-05,
+      "loss": 1.1715,
+      "theoretical_loss": 3.4569584483625233,
+      "tokens_seen": 2013331456
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.875942866313593e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.4569398633435444,
+      "tokens_seen": 2013462528
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.875140426897769e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4569212798731006,
+      "tokens_seen": 2013593600
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.874337987481945e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.456902697950963,
+      "tokens_seen": 2013724672
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.873535548066121e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.456884117576902,
+      "tokens_seen": 2013855744
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.872733108650297e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4568655387506873,
+      "tokens_seen": 2013986816
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.871930669234473e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.4568469614720896,
+      "tokens_seen": 2014117888
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.87112822981865e-05,
+      "loss": 1.3324,
+      "theoretical_loss": 3.456828385740879,
+      "tokens_seen": 2014248960
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.870325790402825e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4568098115568264,
+      "tokens_seen": 2014380032
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.869523350987e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.4567912389197017,
+      "tokens_seen": 2014511104
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.868720911571177e-05,
+      "loss": 1.2964,
+      "theoretical_loss": 3.4567726678292754,
+      "tokens_seen": 2014642176
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.867918472155352e-05,
+      "loss": 1.3277,
+      "theoretical_loss": 3.456754098285319,
+      "tokens_seen": 2014773248
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.867116032739529e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4567355302876024,
+      "tokens_seen": 2014904320
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.866313593323704e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.456716963835896,
+      "tokens_seen": 2015035392
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.865511153907881e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.456698398929971,
+      "tokens_seen": 2015166464
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.864708714492056e-05,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4566798355695987,
+      "tokens_seen": 2015297536
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.863906275076233e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.4566612737545483,
+      "tokens_seen": 2015428608
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.863103835660408e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4566427134845923,
+      "tokens_seen": 2015559680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.862301396244583e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.456624154759501,
+      "tokens_seen": 2015690752
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.86149895682876e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.456605597579045,
+      "tokens_seen": 2015821824
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.860696517412935e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.456587041942996,
+      "tokens_seen": 2015952896
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.859894077997112e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4565684878511247,
+      "tokens_seen": 2016083968
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.859091638581287e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.456549935303202,
+      "tokens_seen": 2016215040
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.858289199165464e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.4565313842989998,
+      "tokens_seen": 2016346112
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.857486759749639e-05,
+      "loss": 1.2602,
+      "theoretical_loss": 3.4565128348382883,
+      "tokens_seen": 2016477184
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.856684320333815e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.4564942869208397,
+      "tokens_seen": 2016608256
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.855881880917991e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.456475740546425,
+      "tokens_seen": 2016739328
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.855079441502167e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4564571957148154,
+      "tokens_seen": 2016870400
+    },
+    {
+      "epoch": 0.22,
+      "objective/train/docs_used": 678408,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2186763286590576,
+      "objective/train/theoretical_loss": 3.456438652425782,
+      "objective/train/tokens_used": 387460576,
+      "theoretical_loss": 3.456438652425782,
+      "tokens_seen": 2017001472
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.854277002086343e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.456438652425782,
+      "tokens_seen": 2017001472
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.853474562670519e-05,
+      "loss": 1.2723,
+      "theoretical_loss": 3.456420110679097,
+      "tokens_seen": 2017132544
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.852672123254695e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4564015704745312,
+      "tokens_seen": 2017263616
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.85186968383887e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.456383031811857,
+      "tokens_seen": 2017394688
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.851067244423046e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.4563644946908454,
+      "tokens_seen": 2017525760
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.850264805007222e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.456345959111268,
+      "tokens_seen": 2017656832
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.849462365591398e-05,
+      "loss": 1.2913,
+      "theoretical_loss": 3.456327425072897,
+      "tokens_seen": 2017787904
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.848659926175574e-05,
+      "loss": 1.2987,
+      "theoretical_loss": 3.4563088925755032,
+      "tokens_seen": 2017918976
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.84785748675975e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4562903616188594,
+      "tokens_seen": 2018050048
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.847055047343926e-05,
+      "loss": 1.3012,
+      "theoretical_loss": 3.456271832202737,
+      "tokens_seen": 2018181120
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.846252607928102e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.456253304326908,
+      "tokens_seen": 2018312192
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.845450168512277e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.456234777991144,
+      "tokens_seen": 2018443264
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.844647729096454e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.4562162531952176,
+      "tokens_seen": 2018574336
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.843845289680629e-05,
+      "loss": 1.2963,
+      "theoretical_loss": 3.456197729938901,
+      "tokens_seen": 2018705408
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.843042850264806e-05,
+      "loss": 1.2124,
+      "theoretical_loss": 3.4561792082219647,
+      "tokens_seen": 2018836480
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.842240410848981e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.4561606880441826,
+      "tokens_seen": 2018967552
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.841437971433158e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.456142169405326,
+      "tokens_seen": 2019098624
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.840635532017333e-05,
+      "loss": 1.3091,
+      "theoretical_loss": 3.456123652305168,
+      "tokens_seen": 2019229696
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.839833092601508e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4561051367434796,
+      "tokens_seen": 2019360768
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.839030653185685e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.4560866227200338,
+      "tokens_seen": 2019491840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.83822821376986e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.4560681102346034,
+      "tokens_seen": 2019622912
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.837425774354037e-05,
+      "loss": 1.2981,
+      "theoretical_loss": 3.4560495992869598,
+      "tokens_seen": 2019753984
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.836623334938212e-05,
+      "loss": 1.3237,
+      "theoretical_loss": 3.456031089876877,
+      "tokens_seen": 2019885056
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.835820895522389e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4560125820041256,
+      "tokens_seen": 2020016128
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.835018456106564e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.45599407566848,
+      "tokens_seen": 2020147200
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.83421601669074e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.455975570869711,
+      "tokens_seen": 2020278272
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.833413577274916e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.455957067607593,
+      "tokens_seen": 2020409344
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.832611137859092e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.4559385658818984,
+      "tokens_seen": 2020540416
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.831808698443268e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.455920065692399,
+      "tokens_seen": 2020671488
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.831006259027444e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.455901567038868,
+      "tokens_seen": 2020802560
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.83020381961162e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.4558830699210787,
+      "tokens_seen": 2020933632
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.829401380195796e-05,
+      "loss": 1.3359,
+      "theoretical_loss": 3.4558645743388037,
+      "tokens_seen": 2021064704
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 7.828598940779972e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.455846080291816,
+      "tokens_seen": 2021195776
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.827796501364148e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.455827587779889,
+      "tokens_seen": 2021326848
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.826994061948323e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.4558090968027946,
+      "tokens_seen": 2021457920
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.8261916225325e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4557906073603073,
+      "tokens_seen": 2021588992
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.825389183116675e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.4557721194521998,
+      "tokens_seen": 2021720064
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.824586743700851e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4557536330782446,
+      "tokens_seen": 2021851136
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.823784304285027e-05,
+      "loss": 1.3111,
+      "theoretical_loss": 3.4557351482382153,
+      "tokens_seen": 2021982208
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.822981864869203e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4557166649318862,
+      "tokens_seen": 2022113280
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.822179425453379e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.455698183159029,
+      "tokens_seen": 2022244352
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.821376986037554e-05,
+      "loss": 1.2785,
+      "theoretical_loss": 3.4556797029194186,
+      "tokens_seen": 2022375424
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.820574546621731e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.455661224212828,
+      "tokens_seen": 2022506496
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.819772107205906e-05,
+      "loss": 1.1786,
+      "theoretical_loss": 3.45564274703903,
+      "tokens_seen": 2022637568
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.818969667790083e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.4556242713977983,
+      "tokens_seen": 2022768640
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.818167228374258e-05,
+      "loss": 1.3333,
+      "theoretical_loss": 3.455605797288907,
+      "tokens_seen": 2022899712
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.817364788958435e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.4555873247121296,
+      "tokens_seen": 2023030784
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.81656234954261e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.45556885366724,
+      "tokens_seen": 2023161856
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.815759910126785e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.4555503841540114,
+      "tokens_seen": 2023292928
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.814957470710962e-05,
+      "loss": 1.1801,
+      "theoretical_loss": 3.4555319161722178,
+      "tokens_seen": 2023424000
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 680686,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2251842021942139,
+      "objective/train/theoretical_loss": 3.455513449721633,
+      "objective/train/tokens_used": 394014176,
+      "theoretical_loss": 3.455513449721633,
+      "tokens_seen": 2023555072
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.814155031295137e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.455513449721633,
+      "tokens_seen": 2023555072
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.813352591879314e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4554949848020313,
+      "tokens_seen": 2023686144
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.812550152463489e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.455476521413186,
+      "tokens_seen": 2023817216
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.811747713047666e-05,
+      "loss": 1.3052,
+      "theoretical_loss": 3.4554580595548714,
+      "tokens_seen": 2023948288
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.810945273631841e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.455439599226861,
+      "tokens_seen": 2024079360
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.810142834216017e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.45542114042893,
+      "tokens_seen": 2024210432
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.809340394800193e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4554026831608518,
+      "tokens_seen": 2024341504
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.808537955384369e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.4553842274224005,
+      "tokens_seen": 2024472576
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.807735515968545e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.45536577321335,
+      "tokens_seen": 2024603648
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.80693307655272e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.455347320533475,
+      "tokens_seen": 2024734720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.806130637136897e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.45532886938255,
+      "tokens_seen": 2024865792
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.805328197721073e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.455310419760349,
+      "tokens_seen": 2024996864
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.804525758305248e-05,
+      "loss": 1.3033,
+      "theoretical_loss": 3.455291971666647,
+      "tokens_seen": 2025127936
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.803723318889425e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.4552735251012177,
+      "tokens_seen": 2025259008
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.8029208794736e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4552550800638357,
+      "tokens_seen": 2025390080
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.802118440057776e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.455236636554276,
+      "tokens_seen": 2025521152
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.801316000641952e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.455218194572313,
+      "tokens_seen": 2025652224
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.800513561226128e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4551997541177206,
+      "tokens_seen": 2025783296
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.799711121810304e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4551813151902753,
+      "tokens_seen": 2025914368
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.79890868239448e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.45516287778975,
+      "tokens_seen": 2026045440
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.798106242978656e-05,
+      "loss": 1.3389,
+      "theoretical_loss": 3.45514444191592,
+      "tokens_seen": 2026176512
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.797303803562831e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.455126007568561,
+      "tokens_seen": 2026307584
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.796501364147008e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.4551075747474465,
+      "tokens_seen": 2026438656
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.795698924731183e-05,
+      "loss": 1.33,
+      "theoretical_loss": 3.4550891434523523,
+      "tokens_seen": 2026569728
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.79489648531536e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.4550707136830536,
+      "tokens_seen": 2026700800
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.794094045899535e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.455052285439325,
+      "tokens_seen": 2026831872
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.793291606483712e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4550338587209413,
+      "tokens_seen": 2026962944
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.792489167067887e-05,
+      "loss": 1.3311,
+      "theoretical_loss": 3.455015433527678,
+      "tokens_seen": 2027094016
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.791686727652062e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.4549970098593104,
+      "tokens_seen": 2027225088
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.790884288236239e-05,
+      "loss": 1.3148,
+      "theoretical_loss": 3.4549785877156136,
+      "tokens_seen": 2027356160
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.790081848820414e-05,
+      "loss": 1.3137,
+      "theoretical_loss": 3.4549601670963623,
+      "tokens_seen": 2027487232
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.789279409404591e-05,
+      "loss": 1.2731,
+      "theoretical_loss": 3.4549417480013327,
+      "tokens_seen": 2027618304
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.788476969988766e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.4549233304302995,
+      "tokens_seen": 2027749376
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.787674530572943e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.4549049143830386,
+      "tokens_seen": 2027880448
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.786872091157118e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.454886499859325,
+      "tokens_seen": 2028011520
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.786069651741294e-05,
+      "loss": 1.2879,
+      "theoretical_loss": 3.4548680868589345,
+      "tokens_seen": 2028142592
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.78526721232547e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.454849675381643,
+      "tokens_seen": 2028273664
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.784464772909646e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.454831265427225,
+      "tokens_seen": 2028404736
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.783662333493822e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4548128569954573,
+      "tokens_seen": 2028535808
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.782859894077998e-05,
+      "loss": 1.3041,
+      "theoretical_loss": 3.4547944500861147,
+      "tokens_seen": 2028666880
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.782057454662174e-05,
+      "loss": 1.3054,
+      "theoretical_loss": 3.4547760446989737,
+      "tokens_seen": 2028797952
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.78125501524635e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.45475764083381,
+      "tokens_seen": 2028929024
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.780452575830525e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.454739238490398,
+      "tokens_seen": 2029060096
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.779650136414702e-05,
+      "loss": 1.3098,
+      "theoretical_loss": 3.454720837668516,
+      "tokens_seen": 2029191168
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.778847696998877e-05,
+      "loss": 1.3001,
+      "theoretical_loss": 3.454702438367938,
+      "tokens_seen": 2029322240
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.778045257583053e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.454684040588441,
+      "tokens_seen": 2029453312
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.777242818167229e-05,
+      "loss": 1.3531,
+      "theoretical_loss": 3.4546656443298005,
+      "tokens_seen": 2029584384
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.776440378751405e-05,
+      "loss": 1.3496,
+      "theoretical_loss": 3.454647249591793,
+      "tokens_seen": 2029715456
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.775637939335581e-05,
+      "loss": 1.2777,
+      "theoretical_loss": 3.454628856374194,
+      "tokens_seen": 2029846528
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.774835499919756e-05,
+      "loss": 1.2855,
+      "theoretical_loss": 3.45461046467678,
+      "tokens_seen": 2029977600
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 682912,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.4055430889129639,
+      "objective/train/theoretical_loss": 3.4545920744993275,
+      "objective/train/tokens_used": 400567776,
+      "theoretical_loss": 3.4545920744993275,
+      "tokens_seen": 2030108672
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.774033060503933e-05,
+      "loss": 1.3464,
+      "theoretical_loss": 3.4545920744993275,
+      "tokens_seen": 2030108672
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.773230621088108e-05,
+      "loss": 1.3124,
+      "theoretical_loss": 3.4545736858416127,
+      "tokens_seen": 2030239744
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.772428181672285e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4545552987034114,
+      "tokens_seen": 2030370816
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.77162574225646e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.4545369130845,
+      "tokens_seen": 2030501888
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.770823302840637e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.454518528984656,
+      "tokens_seen": 2030632960
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.770020863424812e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.4545001464036544,
+      "tokens_seen": 2030764032
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.769218424008987e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.4544817653412725,
+      "tokens_seen": 2030895104
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.768415984593164e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.4544633857972866,
+      "tokens_seen": 2031026176
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.76761354517734e-05,
+      "loss": 1.3233,
+      "theoretical_loss": 3.454445007771474,
+      "tokens_seen": 2031157248
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.766811105761516e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.45442663126361,
+      "tokens_seen": 2031288320
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.766008666345691e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4544082562734717,
+      "tokens_seen": 2031419392
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.765206226929868e-05,
+      "loss": 1.2978,
+      "theoretical_loss": 3.454389882800837,
+      "tokens_seen": 2031550464
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.764403787514043e-05,
+      "loss": 1.3073,
+      "theoretical_loss": 3.4543715108454816,
+      "tokens_seen": 2031681536
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.76360134809822e-05,
+      "loss": 1.3821,
+      "theoretical_loss": 3.454353140407182,
+      "tokens_seen": 2031812608
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.762798908682395e-05,
+      "loss": 1.3261,
+      "theoretical_loss": 3.4543347714857164,
+      "tokens_seen": 2031943680
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.76199646926657e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.4543164040808603,
+      "tokens_seen": 2032074752
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.761194029850747e-05,
+      "loss": 1.2984,
+      "theoretical_loss": 3.4542980381923916,
+      "tokens_seen": 2032205824
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.760391590434923e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.4542796738200874,
+      "tokens_seen": 2032336896
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.759589151019099e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.454261310963724,
+      "tokens_seen": 2032467968
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.758786711603275e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4542429496230787,
+      "tokens_seen": 2032599040
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.757984272187451e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.454224589797929,
+      "tokens_seen": 2032730112
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.757181832771627e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.4542062314880524,
+      "tokens_seen": 2032861184
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.756379393355802e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.4541878746932255,
+      "tokens_seen": 2032992256
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.755576953939979e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4541695194132256,
+      "tokens_seen": 2033123328
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.754774514524154e-05,
+      "loss": 1.3383,
+      "theoretical_loss": 3.454151165647831,
+      "tokens_seen": 2033254400
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.75397207510833e-05,
+      "loss": 1.2948,
+      "theoretical_loss": 3.454132813396818,
+      "tokens_seen": 2033385472
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.753169635692506e-05,
+      "loss": 1.298,
+      "theoretical_loss": 3.454114462659964,
+      "tokens_seen": 2033516544
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.752367196276682e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.454096113437047,
+      "tokens_seen": 2033647616
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.751564756860858e-05,
+      "loss": 1.3188,
+      "theoretical_loss": 3.4540777657278445,
+      "tokens_seen": 2033778688
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.750762317445033e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4540594195321344,
+      "tokens_seen": 2033909760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.74995987802921e-05,
+      "loss": 1.3567,
+      "theoretical_loss": 3.4540410748496937,
+      "tokens_seen": 2034040832
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.749157438613385e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.4540227316803005,
+      "tokens_seen": 2034171904
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.748354999197562e-05,
+      "loss": 1.3104,
+      "theoretical_loss": 3.4540043900237327,
+      "tokens_seen": 2034302976
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.747552559781737e-05,
+      "loss": 1.3663,
+      "theoretical_loss": 3.453986049879767,
+      "tokens_seen": 2034434048
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.746750120365914e-05,
+      "loss": 1.3208,
+      "theoretical_loss": 3.4539677112481826,
+      "tokens_seen": 2034565120
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.745947680950089e-05,
+      "loss": 1.2873,
+      "theoretical_loss": 3.453949374128756,
+      "tokens_seen": 2034696192
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.745145241534264e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.453931038521266,
+      "tokens_seen": 2034827264
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.744342802118441e-05,
+      "loss": 1.2849,
+      "theoretical_loss": 3.453912704425491,
+      "tokens_seen": 2034958336
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.743540362702616e-05,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4538943718412076,
+      "tokens_seen": 2035089408
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.742737923286793e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.453876040768195,
+      "tokens_seen": 2035220480
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.741935483870968e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.4538577112062314,
+      "tokens_seen": 2035351552
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.741133044455145e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.453839383155094,
+      "tokens_seen": 2035482624
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.74033060503932e-05,
+      "loss": 1.2844,
+      "theoretical_loss": 3.453821056614562,
+      "tokens_seen": 2035613696
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.739528165623496e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.4538027315844126,
+      "tokens_seen": 2035744768
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.738725726207672e-05,
+      "loss": 1.2854,
+      "theoretical_loss": 3.4537844080644247,
+      "tokens_seen": 2035875840
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.737923286791848e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.453766086054377,
+      "tokens_seen": 2036006912
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.737120847376024e-05,
+      "loss": 1.3452,
+      "theoretical_loss": 3.4537477655540467,
+      "tokens_seen": 2036137984
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.7363184079602e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.4537294465632136,
+      "tokens_seen": 2036269056
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.735515968544376e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.453711129081655,
+      "tokens_seen": 2036400128
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.734713529128552e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.4536928131091504,
+      "tokens_seen": 2036531200
+    },
+    {
+      "epoch": 0.23,
+      "objective/train/docs_used": 684955,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2071882486343384,
+      "objective/train/theoretical_loss": 3.4536744986454777,
+      "objective/train/tokens_used": 407121376,
+      "theoretical_loss": 3.4536744986454777,
+      "tokens_seen": 2036662272
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.733911089712728e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.4536744986454777,
+      "tokens_seen": 2036662272
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.733108650296904e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.4536561856904155,
+      "tokens_seen": 2036793344
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.732306210881079e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.4536378742437432,
+      "tokens_seen": 2036924416
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.731503771465256e-05,
+      "loss": 1.2945,
+      "theoretical_loss": 3.4536195643052388,
+      "tokens_seen": 2037055488
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.730701332049431e-05,
+      "loss": 1.2436,
+      "theoretical_loss": 3.4536012558746814,
+      "tokens_seen": 2037186560
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.729898892633608e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.4535829489518495,
+      "tokens_seen": 2037317632
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.729096453217783e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.453564643536522,
+      "tokens_seen": 2037448704
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.72829401380196e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4535463396284785,
+      "tokens_seen": 2037579776
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 7.727491574386135e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.453528037227497,
+      "tokens_seen": 2037710848
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.72668913497031e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4535097363333565,
+      "tokens_seen": 2037841920
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.725886695554487e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.4534914369458365,
+      "tokens_seen": 2037972992
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.725084256138662e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.453473139064716,
+      "tokens_seen": 2038104064
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.724281816722839e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4534548426897747,
+      "tokens_seen": 2038235136
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.723479377307014e-05,
+      "loss": 1.2795,
+      "theoretical_loss": 3.4534365478207905,
+      "tokens_seen": 2038366208
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.72267693789119e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.453418254457543,
+      "tokens_seen": 2038497280
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.721874498475366e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.4533999625998124,
+      "tokens_seen": 2038628352
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.721072059059541e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.453381672247377,
+      "tokens_seen": 2038759424
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.720269619643718e-05,
+      "loss": 1.3362,
+      "theoretical_loss": 3.453363383400017,
+      "tokens_seen": 2038890496
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.719467180227893e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.453345096057511,
+      "tokens_seen": 2039021568
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.71866474081207e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.4533268102196386,
+      "tokens_seen": 2039152640
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.717862301396245e-05,
+      "loss": 1.1777,
+      "theoretical_loss": 3.4533085258861798,
+      "tokens_seen": 2039283712
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.71705986198042e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4532902430569132,
+      "tokens_seen": 2039414784
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.716257422564597e-05,
+      "loss": 1.2731,
+      "theoretical_loss": 3.4532719617316197,
+      "tokens_seen": 2039545856
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.715454983148773e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.453253681910078,
+      "tokens_seen": 2039676928
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.714652543732949e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4532354035920685,
+      "tokens_seen": 2039808000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.713850104317125e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.45321712677737,
+      "tokens_seen": 2039939072
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.7130476649013e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.4531988514657628,
+      "tokens_seen": 2040070144
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.712245225485477e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.453180577657027,
+      "tokens_seen": 2040201216
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.711442786069652e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.4531623053509417,
+      "tokens_seen": 2040332288
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.710640346653829e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4531440345472877,
+      "tokens_seen": 2040463360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.709837907238004e-05,
+      "loss": 1.2987,
+      "theoretical_loss": 3.453125765245844,
+      "tokens_seen": 2040594432
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.70903546782218e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.453107497446392,
+      "tokens_seen": 2040725504
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.708233028406356e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.45308923114871,
+      "tokens_seen": 2040856576
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.707430588990531e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.453070966352579,
+      "tokens_seen": 2040987648
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.706628149574708e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.4530527030577796,
+      "tokens_seen": 2041118720
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.705825710158883e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.4530344412640916,
+      "tokens_seen": 2041249792
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.70502327074306e-05,
+      "loss": 1.2906,
+      "theoretical_loss": 3.453016180971295,
+      "tokens_seen": 2041380864
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.704220831327235e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4529979221791707,
+      "tokens_seen": 2041511936
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.70341839191141e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.452979664887498,
+      "tokens_seen": 2041643008
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.702615952495587e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4529614090960585,
+      "tokens_seen": 2041774080
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.701813513079762e-05,
+      "loss": 1.2841,
+      "theoretical_loss": 3.4529431548046317,
+      "tokens_seen": 2041905152
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.701011073663939e-05,
+      "loss": 1.3497,
+      "theoretical_loss": 3.452924902012999,
+      "tokens_seen": 2042036224
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.700208634248114e-05,
+      "loss": 1.279,
+      "theoretical_loss": 3.4529066507209394,
+      "tokens_seen": 2042167296
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.699406194832291e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.452888400928235,
+      "tokens_seen": 2042298368
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.698603755416466e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4528701526346657,
+      "tokens_seen": 2042429440
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.697801316000642e-05,
+      "loss": 1.2314,
+      "theoretical_loss": 3.452851905840012,
+      "tokens_seen": 2042560512
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.696998876584818e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.452833660544055,
+      "tokens_seen": 2042691584
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.696196437168994e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.452815416746575,
+      "tokens_seen": 2042822656
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.69539399775317e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.452797174447354,
+      "tokens_seen": 2042953728
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.694591558337346e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4527789336461714,
+      "tokens_seen": 2043084800
+    },
+    {
+      "epoch": 0.24,
+      "objective/train/docs_used": 686989,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.303419589996338,
+      "objective/train/theoretical_loss": 3.4527606943428086,
+      "objective/train/tokens_used": 413674976,
+      "theoretical_loss": 3.4527606943428086,
+      "tokens_seen": 2043215872
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.693789118921521e-05,
+      "loss": 1.3296,
+      "theoretical_loss": 3.4527606943428086,
+      "tokens_seen": 2043215872
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.692986679505698e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.452742456537047,
+      "tokens_seen": 2043346944
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.692184240089873e-05,
+      "loss": 1.3429,
+      "theoretical_loss": 3.4527242202286663,
+      "tokens_seen": 2043478016
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.69138180067405e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.452705985417449,
+      "tokens_seen": 2043609088
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.690579361258225e-05,
+      "loss": 1.2832,
+      "theoretical_loss": 3.452687752103176,
+      "tokens_seen": 2043740160
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.689776921842402e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.4526695202856272,
+      "tokens_seen": 2043871232
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.688974482426577e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.452651289964585,
+      "tokens_seen": 2044002304
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.688172043010752e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.4526330611398297,
+      "tokens_seen": 2044133376
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.687369603594929e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4526148338111433,
+      "tokens_seen": 2044264448
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.686567164179104e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.4525966079783075,
+      "tokens_seen": 2044395520
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.685764724763281e-05,
+      "loss": 1.2996,
+      "theoretical_loss": 3.452578383641102,
+      "tokens_seen": 2044526592
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.684962285347456e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.45256016079931,
+      "tokens_seen": 2044657664
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.684159845931632e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.4525419394527113,
+      "tokens_seen": 2044788736
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.683357406515808e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.4525237196010883,
+      "tokens_seen": 2044919808
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.682554967099984e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.452505501244223,
+      "tokens_seen": 2045050880
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.68175252768416e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.452487284381896,
+      "tokens_seen": 2045181952
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.680950088268335e-05,
+      "loss": 1.3483,
+      "theoretical_loss": 3.4524690690138895,
+      "tokens_seen": 2045313024
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.680147648852512e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.452450855139985,
+      "tokens_seen": 2045444096
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.679345209436687e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.4524326427599643,
+      "tokens_seen": 2045575168
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.678542770020863e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.452414431873609,
+      "tokens_seen": 2045706240
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.67774033060504e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4523962224807008,
+      "tokens_seen": 2045837312
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.676937891189215e-05,
+      "loss": 1.3167,
+      "theoretical_loss": 3.4523780145810217,
+      "tokens_seen": 2045968384
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.676135451773391e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.452359808174354,
+      "tokens_seen": 2046099456
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.675333012357567e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.452341603260479,
+      "tokens_seen": 2046230528
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.674530572941742e-05,
+      "loss": 1.3099,
+      "theoretical_loss": 3.4523233998391794,
+      "tokens_seen": 2046361600
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.673728133525919e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.4523051979102366,
+      "tokens_seen": 2046492672
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.672925694110094e-05,
+      "loss": 1.1636,
+      "theoretical_loss": 3.4522869974734327,
+      "tokens_seen": 2046623744
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.672123254694271e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.4522687985285505,
+      "tokens_seen": 2046754816
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.671320815278446e-05,
+      "loss": 1.3641,
+      "theoretical_loss": 3.4522506010753715,
+      "tokens_seen": 2046885888
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.670518375862623e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.452232405113678,
+      "tokens_seen": 2047016960
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.669715936446798e-05,
+      "loss": 1.3064,
+      "theoretical_loss": 3.4522142106432527,
+      "tokens_seen": 2047148032
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.668913497030975e-05,
+      "loss": 1.2721,
+      "theoretical_loss": 3.4521960176638773,
+      "tokens_seen": 2047279104
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.66811105761515e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4521778261753346,
+      "tokens_seen": 2047410176
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.667308618199325e-05,
+      "loss": 1.1797,
+      "theoretical_loss": 3.4521596361774067,
+      "tokens_seen": 2047541248
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.666506178783502e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4521414476698764,
+      "tokens_seen": 2047672320
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.665703739367677e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4521232606525265,
+      "tokens_seen": 2047803392
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.664901299951854e-05,
+      "loss": 1.2721,
+      "theoretical_loss": 3.4521050751251385,
+      "tokens_seen": 2047934464
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.664098860536029e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.452086891087496,
+      "tokens_seen": 2048065536
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.663296421120206e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.4520687085393806,
+      "tokens_seen": 2048196608
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.662493981704381e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.4520505274805764,
+      "tokens_seen": 2048327680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.661691542288557e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.4520323479108646,
+      "tokens_seen": 2048458752
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.660889102872733e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4520141698300284,
+      "tokens_seen": 2048589824
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.660086663456909e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.4519959932378512,
+      "tokens_seen": 2048720896
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.659284224041085e-05,
+      "loss": 1.3848,
+      "theoretical_loss": 3.451977818134116,
+      "tokens_seen": 2048851968
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.65848178462526e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.4519596445186047,
+      "tokens_seen": 2048983040
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.657679345209437e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.451941472391101,
+      "tokens_seen": 2049114112
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.656876905793612e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.4519233017513877,
+      "tokens_seen": 2049245184
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.656074466377788e-05,
+      "loss": 1.3647,
+      "theoretical_loss": 3.451905132599248,
+      "tokens_seen": 2049376256
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.655272026961964e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.4518869649344643,
+      "tokens_seen": 2049507328
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.65446958754614e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.451868798756821,
+      "tokens_seen": 2049638400
+    },
+    {
+      "epoch": 0.24,
+      "objective/train/docs_used": 689117,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1913034915924072,
+      "objective/train/theoretical_loss": 3.4518506340660995,
+      "objective/train/tokens_used": 420228576,
+      "theoretical_loss": 3.4518506340660995,
+      "tokens_seen": 2049769472
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.653667148130316e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.4518506340660995,
+      "tokens_seen": 2049769472
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.652864708714492e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.451832470862085,
+      "tokens_seen": 2049900544
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.652062269298668e-05,
+      "loss": 1.3352,
+      "theoretical_loss": 3.4518143091445594,
+      "tokens_seen": 2050031616
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.651259829882844e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.4517961489133064,
+      "tokens_seen": 2050162688
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.650457390467019e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.4517779901681096,
+      "tokens_seen": 2050293760
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.649654951051196e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.4517598329087518,
+      "tokens_seen": 2050424832
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.648852511635371e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.4517416771350176,
+      "tokens_seen": 2050555904
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.648050072219548e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.451723522846689,
+      "tokens_seen": 2050686976
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.647247632803723e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.451705370043551,
+      "tokens_seen": 2050818048
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.6464451933879e-05,
+      "loss": 1.3157,
+      "theoretical_loss": 3.451687218725386,
+      "tokens_seen": 2050949120
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.645642753972075e-05,
+      "loss": 1.1973,
+      "theoretical_loss": 3.451669068891978,
+      "tokens_seen": 2051080192
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.64484031455625e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.451650920543111,
+      "tokens_seen": 2051211264
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.644037875140427e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4516327736785684,
+      "tokens_seen": 2051342336
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.643235435724602e-05,
+      "loss": 1.269,
+      "theoretical_loss": 3.4516146282981346,
+      "tokens_seen": 2051473408
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.642432996308779e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.451596484401592,
+      "tokens_seen": 2051604480
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.641630556892954e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4515783419887263,
+      "tokens_seen": 2051735552
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.640828117477131e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.45156020105932,
+      "tokens_seen": 2051866624
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.640025678061306e-05,
+      "loss": 1.3476,
+      "theoretical_loss": 3.4515420616131576,
+      "tokens_seen": 2051997696
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.639223238645483e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.451523923650023,
+      "tokens_seen": 2052128768
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.638420799229658e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.4515057871697,
+      "tokens_seen": 2052259840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.637618359813834e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.4514876521719735,
+      "tokens_seen": 2052390912
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.63681592039801e-05,
+      "loss": 1.3694,
+      "theoretical_loss": 3.451469518656627,
+      "tokens_seen": 2052521984
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.636013480982186e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.4514513866234444,
+      "tokens_seen": 2052653056
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.635211041566362e-05,
+      "loss": 1.2063,
+      "theoretical_loss": 3.4514332560722103,
+      "tokens_seen": 2052784128
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.634408602150538e-05,
+      "loss": 1.3001,
+      "theoretical_loss": 3.451415127002709,
+      "tokens_seen": 2052915200
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.633606162734714e-05,
+      "loss": 1.2931,
+      "theoretical_loss": 3.451396999414725,
+      "tokens_seen": 2053046272
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.63280372331889e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.451378873308042,
+      "tokens_seen": 2053177344
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.632001283903065e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.451360748682445,
+      "tokens_seen": 2053308416
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.631198844487241e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4513426255377184,
+      "tokens_seen": 2053439488
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.630396405071417e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4513245038736464,
+      "tokens_seen": 2053570560
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.629593965655593e-05,
+      "loss": 1.3848,
+      "theoretical_loss": 3.4513063836900137,
+      "tokens_seen": 2053701632
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.628791526239769e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.451288264986605,
+      "tokens_seen": 2053832704
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.627989086823945e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4512701477632044,
+      "tokens_seen": 2053963776
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.627186647408121e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4512520320195974,
+      "tokens_seen": 2054094848
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.626384207992296e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.451233917755568,
+      "tokens_seen": 2054225920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.625581768576473e-05,
+      "loss": 1.3217,
+      "theoretical_loss": 3.451215804970902,
+      "tokens_seen": 2054356992
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.624779329160648e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.4511976936653825,
+      "tokens_seen": 2054488064
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.623976889744825e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4511795838387958,
+      "tokens_seen": 2054619136
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.623174450329e-05,
+      "loss": 1.29,
+      "theoretical_loss": 3.4511614754909257,
+      "tokens_seen": 2054750208
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.622372010913177e-05,
+      "loss": 1.1946,
+      "theoretical_loss": 3.4511433686215582,
+      "tokens_seen": 2054881280
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.621569571497352e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.451125263230478,
+      "tokens_seen": 2055012352
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.620767132081527e-05,
+      "loss": 1.3557,
+      "theoretical_loss": 3.45110715931747,
+      "tokens_seen": 2055143424
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.619964692665704e-05,
+      "loss": 1.3283,
+      "theoretical_loss": 3.451089056882319,
+      "tokens_seen": 2055274496
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.619162253249879e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.451070955924811,
+      "tokens_seen": 2055405568
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.618359813834056e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4510528564447296,
+      "tokens_seen": 2055536640
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.617557374418231e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4510347584418612,
+      "tokens_seen": 2055667712
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.616754935002408e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.4510166619159914,
+      "tokens_seen": 2055798784
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.615952495586583e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4509985668669043,
+      "tokens_seen": 2055929856
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.615150056170759e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.4509804732943863,
+      "tokens_seen": 2056060928
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.614347616754935e-05,
+      "loss": 1.3232,
+      "theoretical_loss": 3.450962381198222,
+      "tokens_seen": 2056192000
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 691218,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.312801480293274,
+      "objective/train/theoretical_loss": 3.4509442905781973,
+      "objective/train/tokens_used": 426782176,
+      "theoretical_loss": 3.4509442905781973,
+      "tokens_seen": 2056323072
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.61354517733911e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.4509442905781973,
+      "tokens_seen": 2056323072
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.612742737923287e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.450926201434098,
+      "tokens_seen": 2056454144
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.611940298507463e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4509081137657085,
+      "tokens_seen": 2056585216
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.611137859091639e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4508900275728154,
+      "tokens_seen": 2056716288
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.610335419675815e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.450871942855204,
+      "tokens_seen": 2056847360
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.609532980259991e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.45085385961266,
+      "tokens_seen": 2056978432
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.608730540844167e-05,
+      "loss": 1.3021,
+      "theoretical_loss": 3.4508357778449694,
+      "tokens_seen": 2057109504
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.607928101428342e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.450817697551917,
+      "tokens_seen": 2057240576
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.607125662012518e-05,
+      "loss": 1.3645,
+      "theoretical_loss": 3.4507996187332894,
+      "tokens_seen": 2057371648
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.606323222596694e-05,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4507815413888725,
+      "tokens_seen": 2057502720
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.60552078318087e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.4507634655184516,
+      "tokens_seen": 2057633792
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.604718343765046e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.450745391121813,
+      "tokens_seen": 2057764864
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.603915904349222e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.4507273181987426,
+      "tokens_seen": 2057895936
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.603113464933398e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.4507092467490272,
+      "tokens_seen": 2058027008
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.602311025517573e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.4506911767724513,
+      "tokens_seen": 2058158080
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.60150858610175e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.450673108268802,
+      "tokens_seen": 2058289152
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.600706146685925e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.4506550412378654,
+      "tokens_seen": 2058420224
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.599903707270102e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.4506369756794277,
+      "tokens_seen": 2058551296
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.599101267854277e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.450618911593275,
+      "tokens_seen": 2058682368
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.598298828438454e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4506008489791933,
+      "tokens_seen": 2058813440
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.597496389022629e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4505827878369697,
+      "tokens_seen": 2058944512
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.596693949606804e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.45056472816639,
+      "tokens_seen": 2059075584
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.595891510190981e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4505466699672405,
+      "tokens_seen": 2059206656
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.595089070775156e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.450528613239308,
+      "tokens_seen": 2059337728
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.594286631359333e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.450510557982379,
+      "tokens_seen": 2059468800
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.593484191943508e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.45049250419624,
+      "tokens_seen": 2059599872
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.592681752527685e-05,
+      "loss": 1.3108,
+      "theoretical_loss": 3.4504744518806776,
+      "tokens_seen": 2059730944
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.59187931311186e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4504564010354786,
+      "tokens_seen": 2059862016
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.591076873696036e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.450438351660429,
+      "tokens_seen": 2059993088
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.590274434280212e-05,
+      "loss": 1.1662,
+      "theoretical_loss": 3.4504203037553163,
+      "tokens_seen": 2060124160
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.589471994864388e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.450402257319927,
+      "tokens_seen": 2060255232
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.588669555448564e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.450384212354048,
+      "tokens_seen": 2060386304
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.58786711603274e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.450366168857466,
+      "tokens_seen": 2060517376
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.587064676616916e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4503481268299683,
+      "tokens_seen": 2060648448
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.586262237201092e-05,
+      "loss": 1.291,
+      "theoretical_loss": 3.4503300862713413,
+      "tokens_seen": 2060779520
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.585459797785267e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.450312047181373,
+      "tokens_seen": 2060910592
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.584657358369444e-05,
+      "loss": 1.3078,
+      "theoretical_loss": 3.4502940095598484,
+      "tokens_seen": 2061041664
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.583854918953619e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.4502759734065567,
+      "tokens_seen": 2061172736
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.583052479537795e-05,
+      "loss": 1.3245,
+      "theoretical_loss": 3.4502579387212844,
+      "tokens_seen": 2061303808
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.582250040121971e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.4502399055038184,
+      "tokens_seen": 2061434880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.581447600706147e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4502218737539456,
+      "tokens_seen": 2061565952
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.580645161290323e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4502038434714546,
+      "tokens_seen": 2061697024
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.579842721874498e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.4501858146561313,
+      "tokens_seen": 2061828096
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.579040282458675e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4501677873077634,
+      "tokens_seen": 2061959168
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.57823784304285e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.4501497614261387,
+      "tokens_seen": 2062090240
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.577435403627027e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4501317370110445,
+      "tokens_seen": 2062221312
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.576632964211202e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.450113714062268,
+      "tokens_seen": 2062352384
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.575830524795379e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.4500956925795974,
+      "tokens_seen": 2062483456
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.575028085379554e-05,
+      "loss": 1.3079,
+      "theoretical_loss": 3.450077672562819,
+      "tokens_seen": 2062614528
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.574225645963731e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.4500596540117217,
+      "tokens_seen": 2062745600
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 693406,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3297017812728882,
+      "objective/train/theoretical_loss": 3.450041636926093,
+      "objective/train/tokens_used": 433335776,
+      "theoretical_loss": 3.450041636926093,
+      "tokens_seen": 2062876672
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.573423206547906e-05,
+      "loss": 1.3235,
+      "theoretical_loss": 3.450041636926093,
+      "tokens_seen": 2062876672
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.572620767132081e-05,
+      "loss": 1.3031,
+      "theoretical_loss": 3.45002362130572,
+      "tokens_seen": 2063007744
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.571818327716258e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.450005607150391,
+      "tokens_seen": 2063138816
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.571015888300433e-05,
+      "loss": 1.3008,
+      "theoretical_loss": 3.449987594459893,
+      "tokens_seen": 2063269888
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.57021344888461e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.449969583234015,
+      "tokens_seen": 2063400960
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.569411009468785e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.4499515734725446,
+      "tokens_seen": 2063532032
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.568608570052962e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.449933565175269,
+      "tokens_seen": 2063663104
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.567806130637137e-05,
+      "loss": 1.3517,
+      "theoretical_loss": 3.449915558341976,
+      "tokens_seen": 2063794176
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.567003691221313e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.449897552972455,
+      "tokens_seen": 2063925248
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.566201251805489e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.449879549066493,
+      "tokens_seen": 2064056320
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.565398812389665e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.4498615466238784,
+      "tokens_seen": 2064187392
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.564596372973841e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.4498435456443994,
+      "tokens_seen": 2064318464
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.563793933558017e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.449825546127844,
+      "tokens_seen": 2064449536
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.562991494142193e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.449807548074001,
+      "tokens_seen": 2064580608
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.562189054726369e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.4497895514826578,
+      "tokens_seen": 2064711680
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.561386615310544e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.4497715563536033,
+      "tokens_seen": 2064842752
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.56058417589472e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.449753562686626,
+      "tokens_seen": 2064973824
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.559781736478896e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.449735570481514,
+      "tokens_seen": 2065104896
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.558979297063073e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.4497175797380555,
+      "tokens_seen": 2065235968
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.558176857647248e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.44969959045604,
+      "tokens_seen": 2065367040
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.557374418231424e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.449681602635255,
+      "tokens_seen": 2065498112
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.5565719788156e-05,
+      "loss": 1.3227,
+      "theoretical_loss": 3.4496636162754895,
+      "tokens_seen": 2065629184
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.555769539399775e-05,
+      "loss": 1.3212,
+      "theoretical_loss": 3.449645631376532,
+      "tokens_seen": 2065760256
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.554967099983952e-05,
+      "loss": 1.3041,
+      "theoretical_loss": 3.449627647938171,
+      "tokens_seen": 2065891328
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.554164660568127e-05,
+      "loss": 1.2466,
+      "theoretical_loss": 3.4496096659601965,
+      "tokens_seen": 2066022400
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.553362221152304e-05,
+      "loss": 1.3191,
+      "theoretical_loss": 3.4495916854423956,
+      "tokens_seen": 2066153472
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.552559781736479e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.449573706384558,
+      "tokens_seen": 2066284544
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.551757342320656e-05,
+      "loss": 1.3057,
+      "theoretical_loss": 3.449555728786472,
+      "tokens_seen": 2066415616
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.550954902904831e-05,
+      "loss": 1.1839,
+      "theoretical_loss": 3.4495377526479274,
+      "tokens_seen": 2066546688
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.550152463489006e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.449519777968712,
+      "tokens_seen": 2066677760
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.549350024073183e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4495018047486163,
+      "tokens_seen": 2066808832
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.548547584657358e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.4494838329874282,
+      "tokens_seen": 2066939904
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.547745145241535e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.4494658626849364,
+      "tokens_seen": 2067070976
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.54694270582571e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.449447893840931,
+      "tokens_seen": 2067202048
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.546140266409887e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.449429926455201,
+      "tokens_seen": 2067333120
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.545337826994062e-05,
+      "loss": 1.2833,
+      "theoretical_loss": 3.4494119605275344,
+      "tokens_seen": 2067464192
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.544535387578239e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4493939960577222,
+      "tokens_seen": 2067595264
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.543732948162414e-05,
+      "loss": 1.2832,
+      "theoretical_loss": 3.449376033045553,
+      "tokens_seen": 2067726336
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.54293050874659e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.449358071490816,
+      "tokens_seen": 2067857408
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.542128069330766e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4493401113933007,
+      "tokens_seen": 2067988480
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.541325629914942e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4493221527527966,
+      "tokens_seen": 2068119552
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.540523190499118e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4493041955690926,
+      "tokens_seen": 2068250624
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.539720751083294e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.449286239841979,
+      "tokens_seen": 2068381696
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.53891831166747e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.449268285571245,
+      "tokens_seen": 2068512768
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.538115872251646e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.44925033275668,
+      "tokens_seen": 2068643840
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.537313432835821e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.449232381398074,
+      "tokens_seen": 2068774912
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.536510993419998e-05,
+      "loss": 1.3082,
+      "theoretical_loss": 3.4492144314952164,
+      "tokens_seen": 2068905984
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.535708554004173e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.4491964830478974,
+      "tokens_seen": 2069037056
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.53490611458835e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.449178536055906,
+      "tokens_seen": 2069168128
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.534103675172525e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.449160590519033,
+      "tokens_seen": 2069299200
+    },
+    {
+      "epoch": 0.25,
+      "objective/train/docs_used": 695669,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.235776662826538,
+      "objective/train/theoretical_loss": 3.4491426464370676,
+      "objective/train/tokens_used": 439889376,
+      "theoretical_loss": 3.4491426464370676,
+      "tokens_seen": 2069430272
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.533301235756701e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.4491426464370676,
+      "tokens_seen": 2069430272
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.532498796340877e-05,
+      "loss": 1.3007,
+      "theoretical_loss": 3.4491247038098,
+      "tokens_seen": 2069561344
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.531696356925052e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.4491067626370198,
+      "tokens_seen": 2069692416
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.530893917509229e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.449088822918517,
+      "tokens_seen": 2069823488
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.530091478093404e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4490708846540823,
+      "tokens_seen": 2069954560
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.529289038677581e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.449052947843505,
+      "tokens_seen": 2070085632
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.528486599261756e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.4490350124865756,
+      "tokens_seen": 2070216704
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.527684159845933e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.449017078583084,
+      "tokens_seen": 2070347776
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.526881720430108e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.448999146132821,
+      "tokens_seen": 2070478848
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.526079281014283e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.4489812151355763,
+      "tokens_seen": 2070609920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.52527684159846e-05,
+      "loss": 1.3558,
+      "theoretical_loss": 3.44896328559114,
+      "tokens_seen": 2070740992
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.524474402182635e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.448945357499303,
+      "tokens_seen": 2070872064
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.523671962766812e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.4489274308598565,
+      "tokens_seen": 2071003136
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.522869523350987e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4489095056725887,
+      "tokens_seen": 2071134208
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.522067083935164e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.448891581937292,
+      "tokens_seen": 2071265280
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.521264644519339e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.4488736596537564,
+      "tokens_seen": 2071396352
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.520462205103515e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4488557388217718,
+      "tokens_seen": 2071527424
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.519659765687691e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4488378194411293,
+      "tokens_seen": 2071658496
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.518857326271867e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.4488199015116194,
+      "tokens_seen": 2071789568
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.518054886856043e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.4488019850330334,
+      "tokens_seen": 2071920640
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.517252447440219e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.4487840700051615,
+      "tokens_seen": 2072051712
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.516450008024395e-05,
+      "loss": 1.3332,
+      "theoretical_loss": 3.4487661564277943,
+      "tokens_seen": 2072182784
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.51564756860857e-05,
+      "loss": 1.1473,
+      "theoretical_loss": 3.448748244300723,
+      "tokens_seen": 2072313856
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.514845129192747e-05,
+      "loss": 1.3162,
+      "theoretical_loss": 3.448730333623738,
+      "tokens_seen": 2072444928
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.514042689776923e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4487124243966307,
+      "tokens_seen": 2072576000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.513240250361098e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.448694516619192,
+      "tokens_seen": 2072707072
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.512437810945275e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.448676610291213,
+      "tokens_seen": 2072838144
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.51163537152945e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.448658705412484,
+      "tokens_seen": 2072969216
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.510832932113627e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.4486408019827968,
+      "tokens_seen": 2073100288
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.510030492697802e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4486229000019426,
+      "tokens_seen": 2073231360
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.509228053281979e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.448604999469712,
+      "tokens_seen": 2073362432
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.508425613866154e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.4485871003858968,
+      "tokens_seen": 2073493504
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.507623174450329e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.4485692027502877,
+      "tokens_seen": 2073624576
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.506820735034506e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.448551306562676,
+      "tokens_seen": 2073755648
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.506018295618681e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4485334118228534,
+      "tokens_seen": 2073886720
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.505215856202858e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.448515518530612,
+      "tokens_seen": 2074017792
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.504413416787033e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4484976266857412,
+      "tokens_seen": 2074148864
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.50361097737121e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.448479736288035,
+      "tokens_seen": 2074279936
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.502808537955385e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.448461847337282,
+      "tokens_seen": 2074411008
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.50200609853956e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4484439598332766,
+      "tokens_seen": 2074542080
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.501203659123737e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4484260737758086,
+      "tokens_seen": 2074673152
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.500401219707912e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4484081891646703,
+      "tokens_seen": 2074804224
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.499598780292089e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4483903059996535,
+      "tokens_seen": 2074935296
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.498796340876264e-05,
+      "loss": 1.3529,
+      "theoretical_loss": 3.4483724242805494,
+      "tokens_seen": 2075066368
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.497993901460441e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.44835454400715,
+      "tokens_seen": 2075197440
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.497191462044616e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.4483366651792475,
+      "tokens_seen": 2075328512
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.496389022628792e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.4483187877966337,
+      "tokens_seen": 2075459584
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.495586583212968e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4483009118591,
+      "tokens_seen": 2075590656
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.494784143797144e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.448283037366438,
+      "tokens_seen": 2075721728
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.49398170438132e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.4482651643184408,
+      "tokens_seen": 2075852800
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 697911,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2126140594482422,
+      "objective/train/theoretical_loss": 3.4482472927148997,
+      "objective/train/tokens_used": 446442976,
+      "theoretical_loss": 3.4482472927148997,
+      "tokens_seen": 2075983872
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.493179264965496e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4482472927148997,
+      "tokens_seen": 2075983872
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.492376825549672e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.448229422555608,
+      "tokens_seen": 2076114944
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.491574386133848e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.4482115538403555,
+      "tokens_seen": 2076246016
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.490771946718023e-05,
+      "loss": 1.2975,
+      "theoretical_loss": 3.448193686568936,
+      "tokens_seen": 2076377088
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.4899695073022e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.4481758207411417,
+      "tokens_seen": 2076508160
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.489167067886375e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.4481579563567646,
+      "tokens_seen": 2076639232
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.488364628470552e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.4481400934155966,
+      "tokens_seen": 2076770304
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.487562189054727e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.4481222319174307,
+      "tokens_seen": 2076901376
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.486759749638904e-05,
+      "loss": 1.2856,
+      "theoretical_loss": 3.4481043718620583,
+      "tokens_seen": 2077032448
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.485957310223079e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.448086513249273,
+      "tokens_seen": 2077163520
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.485154870807254e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.4480686560788665,
+      "tokens_seen": 2077294592
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.484352431391431e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4480508003506314,
+      "tokens_seen": 2077425664
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.483549991975606e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.4480329460643606,
+      "tokens_seen": 2077556736
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.482747552559783e-05,
+      "loss": 1.3028,
+      "theoretical_loss": 3.4480150932198463,
+      "tokens_seen": 2077687808
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.481945113143958e-05,
+      "loss": 1.2906,
+      "theoretical_loss": 3.447997241816881,
+      "tokens_seen": 2077818880
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.481142673728135e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.447979391855258,
+      "tokens_seen": 2077949952
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.48034023431231e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.44796154333477,
+      "tokens_seen": 2078081024
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.479537794896487e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.447943696255209,
+      "tokens_seen": 2078212096
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.478735355480662e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.447925850616368,
+      "tokens_seen": 2078343168
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.477932916064837e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.4479080064180407,
+      "tokens_seen": 2078474240
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.477130476649014e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.447890163660019,
+      "tokens_seen": 2078605312
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.47632803723319e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4478723223420964,
+      "tokens_seen": 2078736384
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.475525597817366e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.4478544824640656,
+      "tokens_seen": 2078867456
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.474723158401541e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.4478366440257195,
+      "tokens_seen": 2078998528
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.473920718985718e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.447818807026852,
+      "tokens_seen": 2079129600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.473118279569893e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.447800971467255,
+      "tokens_seen": 2079260672
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.472315840154069e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4477831373467223,
+      "tokens_seen": 2079391744
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.471513400738245e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.4477653046650474,
+      "tokens_seen": 2079522816
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.47071096132242e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.4477474734220226,
+      "tokens_seen": 2079653888
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.469908521906597e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.447729643617442,
+      "tokens_seen": 2079784960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.469106082490773e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.4477118152510986,
+      "tokens_seen": 2079916032
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.468303643074949e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4476939883227855,
+      "tokens_seen": 2080047104
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.467501203659125e-05,
+      "loss": 1.2855,
+      "theoretical_loss": 3.447676162832297,
+      "tokens_seen": 2080178176
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.4666987642433e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4476583387794255,
+      "tokens_seen": 2080309248
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.465896324827477e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.447640516163965,
+      "tokens_seen": 2080440320
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.465093885411652e-05,
+      "loss": 1.1882,
+      "theoretical_loss": 3.447622694985709,
+      "tokens_seen": 2080571392
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.464291445995829e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.4476048752444513,
+      "tokens_seen": 2080702464
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.463489006580004e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.447587056939985,
+      "tokens_seen": 2080833536
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.46268656716418e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.447569240072104,
+      "tokens_seen": 2080964608
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.461884127748356e-05,
+      "loss": 1.3055,
+      "theoretical_loss": 3.4475514246406025,
+      "tokens_seen": 2081095680
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.461081688332531e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.447533610645273,
+      "tokens_seen": 2081226752
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.460279248916708e-05,
+      "loss": 1.3231,
+      "theoretical_loss": 3.4475157980859104,
+      "tokens_seen": 2081357824
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.459476809500883e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.4474979869623086,
+      "tokens_seen": 2081488896
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.45867437008506e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.4474801772742607,
+      "tokens_seen": 2081619968
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.457871930669235e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4474623690215607,
+      "tokens_seen": 2081751040
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.457069491253412e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.4474445622040033,
+      "tokens_seen": 2081882112
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.456267051837587e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4474267568213817,
+      "tokens_seen": 2082013184
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.455464612421762e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.4474089528734906,
+      "tokens_seen": 2082144256
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.454662173005939e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.4473911503601236,
+      "tokens_seen": 2082275328
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.453859733590114e-05,
+      "loss": 1.2956,
+      "theoretical_loss": 3.447373349281075,
+      "tokens_seen": 2082406400
+    },
+    {
+      "epoch": 0.26,
+      "objective/train/docs_used": 700019,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1999847888946533,
+      "objective/train/theoretical_loss": 3.447355549636139,
+      "objective/train/tokens_used": 452996576,
+      "theoretical_loss": 3.447355549636139,
+      "tokens_seen": 2082537472
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.453057294174291e-05,
+      "loss": 1.2693,
+      "theoretical_loss": 3.447355549636139,
+      "tokens_seen": 2082537472
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.452254854758466e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.44733775142511,
+      "tokens_seen": 2082668544
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.451452415342643e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4473199546477815,
+      "tokens_seen": 2082799616
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.450649975926818e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4473021593039483,
+      "tokens_seen": 2082930688
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.449847536510994e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.447284365393405,
+      "tokens_seen": 2083061760
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.44904509709517e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.447266572915946,
+      "tokens_seen": 2083192832
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.448242657679346e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4472487818713655,
+      "tokens_seen": 2083323904
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.447440218263522e-05,
+      "loss": 1.2873,
+      "theoretical_loss": 3.447230992259458,
+      "tokens_seen": 2083454976
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.446637778847698e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4472132040800174,
+      "tokens_seen": 2083586048
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.445835339431873e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.447195417332839,
+      "tokens_seen": 2083717120
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.44503290001605e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.447177632017717,
+      "tokens_seen": 2083848192
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.444230460600225e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.447159848134447,
+      "tokens_seen": 2083979264
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.443428021184402e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4471420656828227,
+      "tokens_seen": 2084110336
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.442625581768577e-05,
+      "loss": 1.3188,
+      "theoretical_loss": 3.447124284662639,
+      "tokens_seen": 2084241408
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.441823142352754e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.44710650507369,
+      "tokens_seen": 2084372480
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.441020702936929e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.447088726915772,
+      "tokens_seen": 2084503552
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.440218263521104e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.447070950188679,
+      "tokens_seen": 2084634624
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.439415824105281e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.447053174892206,
+      "tokens_seen": 2084765696
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.438613384689456e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.447035401026147,
+      "tokens_seen": 2084896768
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.437810945273633e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.4470176285902987,
+      "tokens_seen": 2085027840
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.437008505857808e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.446999857584455,
+      "tokens_seen": 2085158912
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.436206066441983e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.4469820880084114,
+      "tokens_seen": 2085289984
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.43540362702616e-05,
+      "loss": 1.2934,
+      "theoretical_loss": 3.446964319861963,
+      "tokens_seen": 2085421056
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.434601187610335e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4469465531449046,
+      "tokens_seen": 2085552128
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.433798748194512e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.446928787857032,
+      "tokens_seen": 2085683200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.432996308778687e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4469110239981395,
+      "tokens_seen": 2085814272
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.432193869362864e-05,
+      "loss": 1.1865,
+      "theoretical_loss": 3.4468932615680226,
+      "tokens_seen": 2085945344
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.43139142994704e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4468755005664775,
+      "tokens_seen": 2086076416
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.430588990531215e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.446857740993299,
+      "tokens_seen": 2086207488
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.429786551115391e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.4468399828482816,
+      "tokens_seen": 2086338560
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.428984111699567e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.4468222261312222,
+      "tokens_seen": 2086469632
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.428181672283743e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.446804470841916,
+      "tokens_seen": 2086600704
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.427379232867919e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4467867169801574,
+      "tokens_seen": 2086731776
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.426576793452094e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.4467689645457433,
+      "tokens_seen": 2086862848
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.425774354036271e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.4467512135384686,
+      "tokens_seen": 2086993920
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.424971914620446e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.4467334639581293,
+      "tokens_seen": 2087124992
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.424169475204623e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4467157158045207,
+      "tokens_seen": 2087256064
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.423367035788798e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.446697969077439,
+      "tokens_seen": 2087387136
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.422564596372975e-05,
+      "loss": 1.329,
+      "theoretical_loss": 3.4466802237766796,
+      "tokens_seen": 2087518208
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.42176215695715e-05,
+      "loss": 1.3175,
+      "theoretical_loss": 3.4466624799020384,
+      "tokens_seen": 2087649280
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.420959717541325e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4466447374533113,
+      "tokens_seen": 2087780352
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.420157278125502e-05,
+      "loss": 1.2063,
+      "theoretical_loss": 3.4466269964302945,
+      "tokens_seen": 2087911424
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.419354838709677e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.446609256832783,
+      "tokens_seen": 2088042496
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.418552399293854e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.446591518660574,
+      "tokens_seen": 2088173568
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.417749959878029e-05,
+      "loss": 1.2113,
+      "theoretical_loss": 3.4465737819134628,
+      "tokens_seen": 2088304640
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.416947520462205e-05,
+      "loss": 1.1379,
+      "theoretical_loss": 3.4465560465912457,
+      "tokens_seen": 2088435712
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.416145081046381e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.446538312693719,
+      "tokens_seen": 2088566784
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.415342641630557e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.446520580220678,
+      "tokens_seen": 2088697856
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.414540202214733e-05,
+      "loss": 1.2949,
+      "theoretical_loss": 3.44650284917192,
+      "tokens_seen": 2088828928
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.413737762798909e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.4464851195472406,
+      "tokens_seen": 2088960000
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 702251,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2285046577453613,
+      "objective/train/theoretical_loss": 3.4464673913464363,
+      "objective/train/tokens_used": 459550176,
+      "theoretical_loss": 3.4464673913464363,
+      "tokens_seen": 2089091072
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.412935323383084e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4464673913464363,
+      "tokens_seen": 2089091072
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.41213288396726e-05,
+      "loss": 1.2136,
+      "theoretical_loss": 3.446449664569303,
+      "tokens_seen": 2089222144
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.411330444551436e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.446431939215638,
+      "tokens_seen": 2089353216
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.410528005135612e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4464142152852366,
+      "tokens_seen": 2089484288
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.409725565719788e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.4463964927778963,
+      "tokens_seen": 2089615360
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.408923126303964e-05,
+      "loss": 1.3037,
+      "theoretical_loss": 3.446378771693413,
+      "tokens_seen": 2089746432
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.40812068688814e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.446361052031583,
+      "tokens_seen": 2089877504
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.407318247472315e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.4463433337922034,
+      "tokens_seen": 2090008576
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.406515808056492e-05,
+      "loss": 1.3457,
+      "theoretical_loss": 3.44632561697507,
+      "tokens_seen": 2090139648
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.405713368640667e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.4463079015799805,
+      "tokens_seen": 2090270720
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.404910929224844e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4462901876067313,
+      "tokens_seen": 2090401792
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.404108489809019e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.446272475055119,
+      "tokens_seen": 2090532864
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.403306050393196e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.44625476392494,
+      "tokens_seen": 2090663936
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.402503610977371e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.4462370542159917,
+      "tokens_seen": 2090795008
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.401701171561546e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.4462193459280708,
+      "tokens_seen": 2090926080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.400898732145723e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.446201639060974,
+      "tokens_seen": 2091057152
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.400096292729898e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.4461839336144986,
+      "tokens_seen": 2091188224
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.399293853314075e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.4461662295884414,
+      "tokens_seen": 2091319296
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.39849141389825e-05,
+      "loss": 1.2931,
+      "theoretical_loss": 3.4461485269825993,
+      "tokens_seen": 2091450368
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.397688974482427e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.4461308257967698,
+      "tokens_seen": 2091581440
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.396886535066602e-05,
+      "loss": 1.1564,
+      "theoretical_loss": 3.44611312603075,
+      "tokens_seen": 2091712512
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.396084095650778e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.446095427684336,
+      "tokens_seen": 2091843584
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.395281656234954e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.4460777307573265,
+      "tokens_seen": 2091974656
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.39447921681913e-05,
+      "loss": 1.1935,
+      "theoretical_loss": 3.446060035249517,
+      "tokens_seen": 2092105728
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.393676777403306e-05,
+      "loss": 1.1902,
+      "theoretical_loss": 3.446042341160707,
+      "tokens_seen": 2092236800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.392874337987482e-05,
+      "loss": 1.3159,
+      "theoretical_loss": 3.4460246484906922,
+      "tokens_seen": 2092367872
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.392071898571658e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4460069572392706,
+      "tokens_seen": 2092498944
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.391269459155834e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.4459892674062393,
+      "tokens_seen": 2092630016
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.390467019740009e-05,
+      "loss": 1.317,
+      "theoretical_loss": 3.445971578991396,
+      "tokens_seen": 2092761088
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.389664580324186e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.445953891994538,
+      "tokens_seen": 2092892160
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.388862140908361e-05,
+      "loss": 1.3074,
+      "theoretical_loss": 3.4459362064154626,
+      "tokens_seen": 2093023232
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.388059701492537e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.445918522253968,
+      "tokens_seen": 2093154304
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.387257262076713e-05,
+      "loss": 1.3144,
+      "theoretical_loss": 3.4459008395098514,
+      "tokens_seen": 2093285376
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.38645482266089e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.4458831581829106,
+      "tokens_seen": 2093416448
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.385652383245065e-05,
+      "loss": 1.3251,
+      "theoretical_loss": 3.4458654782729434,
+      "tokens_seen": 2093547520
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.384849943829241e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.445847799779747,
+      "tokens_seen": 2093678592
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.384047504413417e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.44583012270312,
+      "tokens_seen": 2093809664
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.383245064997592e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.4458124470428597,
+      "tokens_seen": 2093940736
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.382442625581769e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.445794772798764,
+      "tokens_seen": 2094071808
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.381640186165944e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.445777099970631,
+      "tokens_seen": 2094202880
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.380837746750121e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4457594285582585,
+      "tokens_seen": 2094333952
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.380035307334296e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.4457417585614447,
+      "tokens_seen": 2094465024
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.379232867918473e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.4457240899799872,
+      "tokens_seen": 2094596096
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.378430428502648e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.445706422813684,
+      "tokens_seen": 2094727168
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.377627989086823e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.4456887570623342,
+      "tokens_seen": 2094858240
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.376825549671e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.445671092725735,
+      "tokens_seen": 2094989312
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.376023110255175e-05,
+      "loss": 1.3167,
+      "theoretical_loss": 3.445653429803685,
+      "tokens_seen": 2095120384
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.375220670839352e-05,
+      "loss": 1.3017,
+      "theoretical_loss": 3.445635768295982,
+      "tokens_seen": 2095251456
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.374418231423527e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.4456181082024253,
+      "tokens_seen": 2095382528
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.373615792007704e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.4456004495228116,
+      "tokens_seen": 2095513600
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 704471,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3040906190872192,
+      "objective/train/theoretical_loss": 3.445582792256941,
+      "objective/train/tokens_used": 466103776,
+      "theoretical_loss": 3.445582792256941,
+      "tokens_seen": 2095644672
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.372813352591879e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.445582792256941,
+      "tokens_seen": 2095644672
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.372010913176055e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.445565136404611,
+      "tokens_seen": 2095775744
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.371208473760231e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4455474819656198,
+      "tokens_seen": 2095906816
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.370406034344407e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.4455298289397662,
+      "tokens_seen": 2096037888
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.369603594928583e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.445512177326849,
+      "tokens_seen": 2096168960
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.368801155512759e-05,
+      "loss": 1.302,
+      "theoretical_loss": 3.445494527126667,
+      "tokens_seen": 2096300032
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.367998716096935e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.4454768783390177,
+      "tokens_seen": 2096431104
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.36719627668111e-05,
+      "loss": 1.3419,
+      "theoretical_loss": 3.445459230963701,
+      "tokens_seen": 2096562176
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.366393837265286e-05,
+      "loss": 1.269,
+      "theoretical_loss": 3.4454415850005145,
+      "tokens_seen": 2096693248
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.365591397849463e-05,
+      "loss": 1.2646,
+      "theoretical_loss": 3.4454239404492575,
+      "tokens_seen": 2096824320
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.364788958433638e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.445406297309729,
+      "tokens_seen": 2096955392
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.363986519017815e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.445388655581727,
+      "tokens_seen": 2097086464
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.36318407960199e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.445371015265051,
+      "tokens_seen": 2097217536
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.362381640186166e-05,
+      "loss": 1.2784,
+      "theoretical_loss": 3.4453533763595003,
+      "tokens_seen": 2097348608
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.361579200770342e-05,
+      "loss": 1.2722,
+      "theoretical_loss": 3.4453357388648733,
+      "tokens_seen": 2097479680
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.360776761354517e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4453181027809685,
+      "tokens_seen": 2097610752
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.359974321938694e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.445300468107586,
+      "tokens_seen": 2097741824
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.359171882522869e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.445282834844524,
+      "tokens_seen": 2097872896
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.358369443107046e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.445265202991582,
+      "tokens_seen": 2098003968
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.357567003691221e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.445247572548559,
+      "tokens_seen": 2098135040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.356764564275398e-05,
+      "loss": 1.3015,
+      "theoretical_loss": 3.445229943515254,
+      "tokens_seen": 2098266112
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.355962124859573e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.4452123158914665,
+      "tokens_seen": 2098397184
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.35515968544375e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.4451946896769963,
+      "tokens_seen": 2098528256
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.354357246027925e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.4451770648716415,
+      "tokens_seen": 2098659328
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.3535548066121e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4451594414752025,
+      "tokens_seen": 2098790400
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.352752367196277e-05,
+      "loss": 1.1785,
+      "theoretical_loss": 3.4451418194874783,
+      "tokens_seen": 2098921472
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.351949927780452e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4451241989082684,
+      "tokens_seen": 2099052544
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.351147488364629e-05,
+      "loss": 1.326,
+      "theoretical_loss": 3.445106579737372,
+      "tokens_seen": 2099183616
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.350345048948804e-05,
+      "loss": 1.2948,
+      "theoretical_loss": 3.445088961974589,
+      "tokens_seen": 2099314688
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.349542609532981e-05,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4450713456197186,
+      "tokens_seen": 2099445760
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.348740170117156e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.4450537306725604,
+      "tokens_seen": 2099576832
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.347937730701332e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.445036117132915,
+      "tokens_seen": 2099707904
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.347135291285508e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.4450185050005806,
+      "tokens_seen": 2099838976
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.346332851869684e-05,
+      "loss": 1.3334,
+      "theoretical_loss": 3.445000894275358,
+      "tokens_seen": 2099970048
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.34553041245386e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.4449832849570465,
+      "tokens_seen": 2100101120
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.344727973038036e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.444965677045446,
+      "tokens_seen": 2100232192
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.343925533622212e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.444948070540357,
+      "tokens_seen": 2100363264
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.343123094206388e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.4449304654415775,
+      "tokens_seen": 2100494336
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.342320654790563e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.4449128617489095,
+      "tokens_seen": 2100625408
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.34151821537474e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.444895259462152,
+      "tokens_seen": 2100756480
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.340715775958915e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.444877658581105,
+      "tokens_seen": 2100887552
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.339913336543092e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.4448600591055687,
+      "tokens_seen": 2101018624
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.339110897127267e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.4448424610353436,
+      "tokens_seen": 2101149696
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.338308457711443e-05,
+      "loss": 1.3713,
+      "theoretical_loss": 3.444824864370229,
+      "tokens_seen": 2101280768
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.337506018295619e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.4448072691100258,
+      "tokens_seen": 2101411840
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.336703578879794e-05,
+      "loss": 1.3606,
+      "theoretical_loss": 3.4447896752545333,
+      "tokens_seen": 2101542912
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.335901139463971e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.444772082803553,
+      "tokens_seen": 2101673984
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.335098700048146e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.4447544917568846,
+      "tokens_seen": 2101805056
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.334296260632323e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.444736902114328,
+      "tokens_seen": 2101936128
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.333493821216498e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.4447193138756838,
+      "tokens_seen": 2102067200
+    },
+    {
+      "epoch": 0.27,
+      "objective/train/docs_used": 706780,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2246379852294922,
+      "objective/train/theoretical_loss": 3.444701727040753,
+      "objective/train/tokens_used": 472657376,
+      "theoretical_loss": 3.444701727040753,
+      "tokens_seen": 2102198272
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.332691381800675e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.444701727040753,
+      "tokens_seen": 2102198272
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.33188894238485e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.4446841416093354,
+      "tokens_seen": 2102329344
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.331086502969025e-05,
+      "loss": 1.3022,
+      "theoretical_loss": 3.444666557581232,
+      "tokens_seen": 2102460416
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.330284063553202e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.444648974956243,
+      "tokens_seen": 2102591488
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.329481624137377e-05,
+      "loss": 1.3069,
+      "theoretical_loss": 3.444631393734169,
+      "tokens_seen": 2102722560
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.328679184721554e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4446138139148106,
+      "tokens_seen": 2102853632
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.32787674530573e-05,
+      "loss": 1.3272,
+      "theoretical_loss": 3.444596235497969,
+      "tokens_seen": 2102984704
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.327074305889906e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.444578658483444,
+      "tokens_seen": 2103115776
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.326271866474081e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.4445610828710373,
+      "tokens_seen": 2103246848
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.325469427058258e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.4445435086605487,
+      "tokens_seen": 2103377920
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.324666987642433e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.4445259358517797,
+      "tokens_seen": 2103508992
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.323864548226609e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.444508364444531,
+      "tokens_seen": 2103640064
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.323062108810785e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.4444907944386034,
+      "tokens_seen": 2103771136
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.32225966939496e-05,
+      "loss": 1.3107,
+      "theoretical_loss": 3.4444732258337982,
+      "tokens_seen": 2103902208
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.321457229979137e-05,
+      "loss": 1.1548,
+      "theoretical_loss": 3.4444556586299155,
+      "tokens_seen": 2104033280
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.320654790563313e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4444380928267577,
+      "tokens_seen": 2104164352
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.319852351147489e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.444420528424125,
+      "tokens_seen": 2104295424
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.319049911731665e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4444029654218182,
+      "tokens_seen": 2104426496
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.31824747231584e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.444385403819639,
+      "tokens_seen": 2104557568
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.317445032900017e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.4443678436173886,
+      "tokens_seen": 2104688640
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.316642593484192e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.444350284814868,
+      "tokens_seen": 2104819712
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.315840154068369e-05,
+      "loss": 1.2817,
+      "theoretical_loss": 3.444332727411879,
+      "tokens_seen": 2104950784
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.315037714652544e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4443151714082223,
+      "tokens_seen": 2105081856
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.31423527523672e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4442976168036994,
+      "tokens_seen": 2105212928
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.313432835820896e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.4442800635981117,
+      "tokens_seen": 2105344000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.312630396405071e-05,
+      "loss": 1.3063,
+      "theoretical_loss": 3.44426251179126,
+      "tokens_seen": 2105475072
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.311827956989248e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.444244961382948,
+      "tokens_seen": 2105606144
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.311025517573423e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4442274123729746,
+      "tokens_seen": 2105737216
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.3102230781576e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4442098647611425,
+      "tokens_seen": 2105868288
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.309420638741775e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.4441923185472536,
+      "tokens_seen": 2105999360
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.308618199325952e-05,
+      "loss": 1.2044,
+      "theoretical_loss": 3.444174773731109,
+      "tokens_seen": 2106130432
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.307815759910127e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.44415723031251,
+      "tokens_seen": 2106261504
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.307013320494302e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4441396882912594,
+      "tokens_seen": 2106392576
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.306210881078479e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.4441221476671586,
+      "tokens_seen": 2106523648
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.305408441662654e-05,
+      "loss": 1.3224,
+      "theoretical_loss": 3.444104608440009,
+      "tokens_seen": 2106654720
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.304606002246831e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.444087070609613,
+      "tokens_seen": 2106785792
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.303803562831006e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.444069534175772,
+      "tokens_seen": 2106916864
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.303001123415183e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4440519991382876,
+      "tokens_seen": 2107047936
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.302198683999358e-05,
+      "loss": 1.3384,
+      "theoretical_loss": 3.444034465496962,
+      "tokens_seen": 2107179008
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.301396244583534e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.444016933251598,
+      "tokens_seen": 2107310080
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.30059380516771e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.4439994024019973,
+      "tokens_seen": 2107441152
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.299791365751886e-05,
+      "loss": 1.2955,
+      "theoretical_loss": 3.4439818729479614,
+      "tokens_seen": 2107572224
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.298988926336062e-05,
+      "loss": 1.167,
+      "theoretical_loss": 3.443964344889293,
+      "tokens_seen": 2107703296
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.298186486920238e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.4439468182257937,
+      "tokens_seen": 2107834368
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.297384047504414e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.4439292929572662,
+      "tokens_seen": 2107965440
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.29658160808859e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.443911769083513,
+      "tokens_seen": 2108096512
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.295779168672765e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.4438942466043354,
+      "tokens_seen": 2108227584
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.294976729256942e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4438767255195364,
+      "tokens_seen": 2108358656
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.294174289841117e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.4438592058289186,
+      "tokens_seen": 2108489728
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.293371850425294e-05,
+      "loss": 1.3578,
+      "theoretical_loss": 3.4438416875322835,
+      "tokens_seen": 2108620800
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 708880,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2966865301132202,
+      "objective/train/theoretical_loss": 3.4438241706294344,
+      "objective/train/tokens_used": 479210976,
+      "theoretical_loss": 3.4438241706294344,
+      "tokens_seen": 2108751872
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.292569411009469e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.4438241706294344,
+      "tokens_seen": 2108751872
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.291766971593646e-05,
+      "loss": 1.2971,
+      "theoretical_loss": 3.4438066551201736,
+      "tokens_seen": 2108882944
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.290964532177821e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4437891410043036,
+      "tokens_seen": 2109014016
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.290162092761998e-05,
+      "loss": 1.3342,
+      "theoretical_loss": 3.4437716282816266,
+      "tokens_seen": 2109145088
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.289359653346173e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.443754116951946,
+      "tokens_seen": 2109276160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.288557213930348e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.4437366070150635,
+      "tokens_seen": 2109407232
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.287754774514525e-05,
+      "loss": 1.3262,
+      "theoretical_loss": 3.4437190984707824,
+      "tokens_seen": 2109538304
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.2869523350987e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.443701591318905,
+      "tokens_seen": 2109669376
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.286149895682877e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.4436840855592346,
+      "tokens_seen": 2109800448
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.285347456267052e-05,
+      "loss": 1.2949,
+      "theoretical_loss": 3.4436665811915734,
+      "tokens_seen": 2109931520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.284545016851229e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.443649078215725,
+      "tokens_seen": 2110062592
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.283742577435404e-05,
+      "loss": 1.1904,
+      "theoretical_loss": 3.4436315766314918,
+      "tokens_seen": 2110193664
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.28294013801958e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.443614076438677,
+      "tokens_seen": 2110324736
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.282137698603756e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.443596577637083,
+      "tokens_seen": 2110455808
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.281335259187931e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.443579080226513,
+      "tokens_seen": 2110586880
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.280532819772108e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.443561584206771,
+      "tokens_seen": 2110717952
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.279730380356283e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4435440895776583,
+      "tokens_seen": 2110849024
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.27892794094046e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.44352659633898,
+      "tokens_seen": 2110980096
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.278125501524635e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.4435091044905377,
+      "tokens_seen": 2111111168
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.27732306210881e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.443491614032135,
+      "tokens_seen": 2111242240
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.276520622692987e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.443474124963576,
+      "tokens_seen": 2111373312
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.275718183277163e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.4434566372846636,
+      "tokens_seen": 2111504384
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.274915743861339e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.4434391509952,
+      "tokens_seen": 2111635456
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.274113304445515e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.4434216660949897,
+      "tokens_seen": 2111766528
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.273310865029691e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.443404182583836,
+      "tokens_seen": 2111897600
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.272508425613867e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4433867004615424,
+      "tokens_seen": 2112028672
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.271705986198042e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.443369219727912,
+      "tokens_seen": 2112159744
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.270903546782219e-05,
+      "loss": 1.3222,
+      "theoretical_loss": 3.4433517403827487,
+      "tokens_seen": 2112290816
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.270101107366394e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.4433342624258563,
+      "tokens_seen": 2112421888
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.26929866795057e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.4433167858570375,
+      "tokens_seen": 2112552960
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.268496228534746e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.443299310676096,
+      "tokens_seen": 2112684032
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.267693789118923e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4432818368828366,
+      "tokens_seen": 2112815104
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.266891349703098e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.443264364477062,
+      "tokens_seen": 2112946176
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.266088910287273e-05,
+      "loss": 1.3329,
+      "theoretical_loss": 3.4432468934585767,
+      "tokens_seen": 2113077248
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.26528647087145e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.443229423827184,
+      "tokens_seen": 2113208320
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.264484031455625e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.4432119555826883,
+      "tokens_seen": 2113339392
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.263681592039802e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.4431944887248926,
+      "tokens_seen": 2113470464
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.262879152623977e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.443177023253601,
+      "tokens_seen": 2113601536
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.262076713208154e-05,
+      "loss": 1.3261,
+      "theoretical_loss": 3.4431595591686186,
+      "tokens_seen": 2113732608
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.261274273792329e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4431420964697486,
+      "tokens_seen": 2113863680
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.260471834376506e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4431246351567943,
+      "tokens_seen": 2113994752
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.259669394960681e-05,
+      "loss": 1.3501,
+      "theoretical_loss": 3.443107175229561,
+      "tokens_seen": 2114125824
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.258866955544856e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4430897166878527,
+      "tokens_seen": 2114256896
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.258064516129033e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.4430722595314727,
+      "tokens_seen": 2114387968
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.257262076713208e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.4430548037602255,
+      "tokens_seen": 2114519040
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.256459637297385e-05,
+      "loss": 1.3206,
+      "theoretical_loss": 3.4430373493739164,
+      "tokens_seen": 2114650112
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.25565719788156e-05,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4430198963723484,
+      "tokens_seen": 2114781184
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.254854758465737e-05,
+      "loss": 1.3142,
+      "theoretical_loss": 3.4430024447553267,
+      "tokens_seen": 2114912256
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.254052319049912e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4429849945226545,
+      "tokens_seen": 2115043328
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.253249879634088e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.4429675456741373,
+      "tokens_seen": 2115174400
+    },
+    {
+      "epoch": 0.28,
+      "objective/train/docs_used": 711117,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2823063135147095,
+      "objective/train/theoretical_loss": 3.4429500982095798,
+      "objective/train/tokens_used": 485764576,
+      "theoretical_loss": 3.4429500982095798,
+      "tokens_seen": 2115305472
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.252447440218264e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4429500982095798,
+      "tokens_seen": 2115305472
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.25164500080244e-05,
+      "loss": 1.3067,
+      "theoretical_loss": 3.4429326521287855,
+      "tokens_seen": 2115436544
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.250842561386616e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.442915207431559,
+      "tokens_seen": 2115567616
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.250040121970792e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.4428977641177054,
+      "tokens_seen": 2115698688
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.249237682554968e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.4428803221870297,
+      "tokens_seen": 2115829760
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.248435243139144e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.4428628816393347,
+      "tokens_seen": 2115960832
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.247632803723319e-05,
+      "loss": 1.3595,
+      "theoretical_loss": 3.4428454424744275,
+      "tokens_seen": 2116091904
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.246830364307496e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.442828004692111,
+      "tokens_seen": 2116222976
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.246027924891671e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.4428105682921912,
+      "tokens_seen": 2116354048
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.245225485475848e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.442793133274472,
+      "tokens_seen": 2116485120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.244423046060023e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.4427756996387586,
+      "tokens_seen": 2116616192
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.2436206066442e-05,
+      "loss": 1.3083,
+      "theoretical_loss": 3.442758267384856,
+      "tokens_seen": 2116747264
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.242818167228375e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.4427408365125687,
+      "tokens_seen": 2116878336
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.24201572781255e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.442723407021702,
+      "tokens_seen": 2117009408
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.241213288396727e-05,
+      "loss": 1.3397,
+      "theoretical_loss": 3.442705978912061,
+      "tokens_seen": 2117140480
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.240410848980902e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.4426885521834505,
+      "tokens_seen": 2117271552
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.239608409565079e-05,
+      "loss": 1.3169,
+      "theoretical_loss": 3.4426711268356756,
+      "tokens_seen": 2117402624
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.238805970149254e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.4426537028685416,
+      "tokens_seen": 2117533696
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.238003530733431e-05,
+      "loss": 1.3453,
+      "theoretical_loss": 3.442636280281853,
+      "tokens_seen": 2117664768
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.237201091317606e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4426188590754165,
+      "tokens_seen": 2117795840
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.236398651901781e-05,
+      "loss": 1.2784,
+      "theoretical_loss": 3.442601439249036,
+      "tokens_seen": 2117926912
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.235596212485958e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.442584020802517,
+      "tokens_seen": 2118057984
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.234793773070133e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4425666037356653,
+      "tokens_seen": 2118189056
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.23399133365431e-05,
+      "loss": 1.3062,
+      "theoretical_loss": 3.442549188048285,
+      "tokens_seen": 2118320128
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.233188894238485e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.4425317737401837,
+      "tokens_seen": 2118451200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.232386454822662e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.442514360811165,
+      "tokens_seen": 2118582272
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.231584015406837e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.4424969492610344,
+      "tokens_seen": 2118713344
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.230781575991014e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.442479539089599,
+      "tokens_seen": 2118844416
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.22997913657519e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.4424621302966627,
+      "tokens_seen": 2118975488
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.229176697159365e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4424447228820316,
+      "tokens_seen": 2119106560
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.228374257743541e-05,
+      "loss": 1.3223,
+      "theoretical_loss": 3.442427316845512,
+      "tokens_seen": 2119237632
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.227571818327717e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4424099121869087,
+      "tokens_seen": 2119368704
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.226769378911893e-05,
+      "loss": 1.2963,
+      "theoretical_loss": 3.4423925089060274,
+      "tokens_seen": 2119499776
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.225966939496069e-05,
+      "loss": 1.3461,
+      "theoretical_loss": 3.442375107002675,
+      "tokens_seen": 2119630848
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.225164500080245e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.442357706476656,
+      "tokens_seen": 2119761920
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.22436206066442e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4423403073277763,
+      "tokens_seen": 2119892992
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.223559621248596e-05,
+      "loss": 1.2872,
+      "theoretical_loss": 3.442322909555843,
+      "tokens_seen": 2120024064
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 7.222757181832773e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.442305513160661,
+      "tokens_seen": 2120155136
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.221954742416948e-05,
+      "loss": 1.2602,
+      "theoretical_loss": 3.442288118142036,
+      "tokens_seen": 2120286208
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.221152303001125e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.4422707244997746,
+      "tokens_seen": 2120417280
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.2203498635853e-05,
+      "loss": 1.3049,
+      "theoretical_loss": 3.4422533322336824,
+      "tokens_seen": 2120548352
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.219547424169477e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.442235941343566,
+      "tokens_seen": 2120679424
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.218744984753652e-05,
+      "loss": 1.3453,
+      "theoretical_loss": 3.4422185518292316,
+      "tokens_seen": 2120810496
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.217942545337827e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.4422011636904846,
+      "tokens_seen": 2120941568
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.217140105922004e-05,
+      "loss": 1.3117,
+      "theoretical_loss": 3.4421837769271315,
+      "tokens_seen": 2121072640
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.216337666506179e-05,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4421663915389784,
+      "tokens_seen": 2121203712
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.215535227090356e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.442149007525832,
+      "tokens_seen": 2121334784
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.214732787674531e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.442131624887499,
+      "tokens_seen": 2121465856
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.213930348258708e-05,
+      "loss": 1.2727,
+      "theoretical_loss": 3.4421142436237844,
+      "tokens_seen": 2121596928
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.213127908842883e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.442096863734495,
+      "tokens_seen": 2121728000
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 713301,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.291300654411316,
+      "objective/train/theoretical_loss": 3.442079485219438,
+      "objective/train/tokens_used": 492318176,
+      "theoretical_loss": 3.442079485219438,
+      "tokens_seen": 2121859072
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.212325469427058e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.442079485219438,
+      "tokens_seen": 2121859072
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.211523030011235e-05,
+      "loss": 1.3634,
+      "theoretical_loss": 3.442062108078419,
+      "tokens_seen": 2121990144
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.21072059059541e-05,
+      "loss": 1.317,
+      "theoretical_loss": 3.4420447323112455,
+      "tokens_seen": 2122121216
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.209918151179587e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.4420273579177225,
+      "tokens_seen": 2122252288
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.209115711763762e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.442009984897658,
+      "tokens_seen": 2122383360
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.208313272347939e-05,
+      "loss": 1.2848,
+      "theoretical_loss": 3.441992613250858,
+      "tokens_seen": 2122514432
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.207510832932114e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.4419752429771293,
+      "tokens_seen": 2122645504
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.20670839351629e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.4419578740762784,
+      "tokens_seen": 2122776576
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.205905954100466e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.4419405065481126,
+      "tokens_seen": 2122907648
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.205103514684642e-05,
+      "loss": 1.3022,
+      "theoretical_loss": 3.4419231403924373,
+      "tokens_seen": 2123038720
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.204301075268818e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.441905775609061,
+      "tokens_seen": 2123169792
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.203498635852994e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.44188841219779,
+      "tokens_seen": 2123300864
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.20269619643717e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.44187105015843,
+      "tokens_seen": 2123431936
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.201893757021346e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.44185368949079,
+      "tokens_seen": 2123563008
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.201091317605521e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.4418363301946755,
+      "tokens_seen": 2123694080
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.200288878189698e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.441818972269894,
+      "tokens_seen": 2123825152
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.199486438773873e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.441801615716252,
+      "tokens_seen": 2123956224
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.19868399935805e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.4417842605335576,
+      "tokens_seen": 2124087296
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.197881559942225e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4417669067216172,
+      "tokens_seen": 2124218368
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.197079120526402e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.441749554280238,
+      "tokens_seen": 2124349440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.196276681110577e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4417322032092272,
+      "tokens_seen": 2124480512
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.195474241694754e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.441714853508392,
+      "tokens_seen": 2124611584
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.194671802278929e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.4416975051775407,
+      "tokens_seen": 2124742656
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.193869362863104e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4416801582164793,
+      "tokens_seen": 2124873728
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.193066923447281e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.4416628126250153,
+      "tokens_seen": 2125004800
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.192264484031456e-05,
+      "loss": 1.3571,
+      "theoretical_loss": 3.4416454684029567,
+      "tokens_seen": 2125135872
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.191462044615633e-05,
+      "loss": 1.3138,
+      "theoretical_loss": 3.4416281255501113,
+      "tokens_seen": 2125266944
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.190659605199808e-05,
+      "loss": 1.3148,
+      "theoretical_loss": 3.441610784066285,
+      "tokens_seen": 2125398016
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.189857165783985e-05,
+      "loss": 1.3317,
+      "theoretical_loss": 3.4415934439512865,
+      "tokens_seen": 2125529088
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.18905472636816e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4415761052049234,
+      "tokens_seen": 2125660160
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.188252286952335e-05,
+      "loss": 1.2968,
+      "theoretical_loss": 3.441558767827003,
+      "tokens_seen": 2125791232
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.187449847536512e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.4415414318173325,
+      "tokens_seen": 2125922304
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.186647408120687e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.44152409717572,
+      "tokens_seen": 2126053376
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.185844968704864e-05,
+      "loss": 1.3175,
+      "theoretical_loss": 3.441506763901974,
+      "tokens_seen": 2126184448
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.18504252928904e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.441489431995901,
+      "tokens_seen": 2126315520
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.184240089873216e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.441472101457309,
+      "tokens_seen": 2126446592
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.183437650457391e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.441454772286006,
+      "tokens_seen": 2126577664
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.182635211041567e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4414374444818003,
+      "tokens_seen": 2126708736
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.181832771625743e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.4414201180444994,
+      "tokens_seen": 2126839808
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.181030332209919e-05,
+      "loss": 1.299,
+      "theoretical_loss": 3.4414027929739115,
+      "tokens_seen": 2126970880
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.180227892794095e-05,
+      "loss": 1.3513,
+      "theoretical_loss": 3.441385469269844,
+      "tokens_seen": 2127101952
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.17942545337827e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.4413681469321054,
+      "tokens_seen": 2127233024
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.178623013962446e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.4413508259605035,
+      "tokens_seen": 2127364096
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.177820574546623e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.441333506354847,
+      "tokens_seen": 2127495168
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.177018135130798e-05,
+      "loss": 1.2734,
+      "theoretical_loss": 3.441316188114943,
+      "tokens_seen": 2127626240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.176215695714975e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.4412988712406007,
+      "tokens_seen": 2127757312
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.17541325629915e-05,
+      "loss": 1.3388,
+      "theoretical_loss": 3.441281555731628,
+      "tokens_seen": 2127888384
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.174610816883327e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.4412642415878327,
+      "tokens_seen": 2128019456
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.173808377467502e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.441246928809023,
+      "tokens_seen": 2128150528
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.173005938051677e-05,
+      "loss": 1.3676,
+      "theoretical_loss": 3.4412296173950083,
+      "tokens_seen": 2128281600
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 715451,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2150688171386719,
+      "objective/train/theoretical_loss": 3.4412123073455962,
+      "objective/train/tokens_used": 498871776,
+      "theoretical_loss": 3.4412123073455962,
+      "tokens_seen": 2128412672
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.172203498635854e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.4412123073455962,
+      "tokens_seen": 2128412672
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.171401059220029e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.4411949986605954,
+      "tokens_seen": 2128543744
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.170598619804206e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.4411776913398135,
+      "tokens_seen": 2128674816
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.169796180388381e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.44116038538306,
+      "tokens_seen": 2128805888
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.168993740972557e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.4411430807901433,
+      "tokens_seen": 2128936960
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.168191301556733e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.4411257775608717,
+      "tokens_seen": 2129068032
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.167388862140908e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.441108475695054,
+      "tokens_seen": 2129199104
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.166586422725085e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.441091175192498,
+      "tokens_seen": 2129330176
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.16578398330926e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.4410738760530135,
+      "tokens_seen": 2129461248
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.164981543893436e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.4410565782764087,
+      "tokens_seen": 2129592320
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.164179104477612e-05,
+      "loss": 1.3175,
+      "theoretical_loss": 3.4410392818624924,
+      "tokens_seen": 2129723392
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.163376665061788e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.441021986811074,
+      "tokens_seen": 2129854464
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.162574225645964e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.441004693121961,
+      "tokens_seen": 2129985536
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.16177178623014e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.4409874007949632,
+      "tokens_seen": 2130116608
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.160969346814316e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4409701098298893,
+      "tokens_seen": 2130247680
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.160166907398492e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.440952820226549,
+      "tokens_seen": 2130378752
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.159364467982667e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.44093553198475,
+      "tokens_seen": 2130509824
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.158562028566844e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4409182451043017,
+      "tokens_seen": 2130640896
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.157759589151019e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.4409009595850137,
+      "tokens_seen": 2130771968
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.156957149735196e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.4408836754266945,
+      "tokens_seen": 2130903040
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.156154710319371e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.440866392629154,
+      "tokens_seen": 2131034112
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.155352270903546e-05,
+      "loss": 1.2836,
+      "theoretical_loss": 3.4408491111922004,
+      "tokens_seen": 2131165184
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.154549831487723e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.440831831115643,
+      "tokens_seen": 2131296256
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.153747392071898e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4408145523992926,
+      "tokens_seen": 2131427328
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.152944952656075e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4407972750429567,
+      "tokens_seen": 2131558400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.15214251324025e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.4407799990464456,
+      "tokens_seen": 2131689472
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.151340073824427e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.440762724409568,
+      "tokens_seen": 2131820544
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.150537634408602e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.440745451132134,
+      "tokens_seen": 2131951616
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.149735194992778e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.440728179213952,
+      "tokens_seen": 2132082688
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.148932755576954e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4407109086548333,
+      "tokens_seen": 2132213760
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.14813031616113e-05,
+      "loss": 1.3231,
+      "theoretical_loss": 3.4406936394545857,
+      "tokens_seen": 2132344832
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.147327876745306e-05,
+      "loss": 1.3019,
+      "theoretical_loss": 3.4406763716130193,
+      "tokens_seen": 2132475904
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.146525437329482e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.440659105129944,
+      "tokens_seen": 2132606976
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.145722997913657e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.4406418400051693,
+      "tokens_seen": 2132738048
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.144920558497834e-05,
+      "loss": 1.2924,
+      "theoretical_loss": 3.440624576238505,
+      "tokens_seen": 2132869120
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.144118119082009e-05,
+      "loss": 1.3413,
+      "theoretical_loss": 3.4406073138297604,
+      "tokens_seen": 2133000192
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.143315679666185e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.440590052778745,
+      "tokens_seen": 2133131264
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.142513240250361e-05,
+      "loss": 1.2823,
+      "theoretical_loss": 3.44057279308527,
+      "tokens_seen": 2133262336
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.141710800834537e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.4405555347491443,
+      "tokens_seen": 2133393408
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.140908361418713e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.440538277770177,
+      "tokens_seen": 2133524480
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.140105922002888e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.4405210221481797,
+      "tokens_seen": 2133655552
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.139303482587065e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.4405037678829604,
+      "tokens_seen": 2133786624
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.13850104317124e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.440486514974331,
+      "tokens_seen": 2133917696
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.137698603755417e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.4404692634221004,
+      "tokens_seen": 2134048768
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.136896164339592e-05,
+      "loss": 1.3442,
+      "theoretical_loss": 3.440452013226079,
+      "tokens_seen": 2134179840
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.136093724923767e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.440434764386077,
+      "tokens_seen": 2134310912
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.135291285507944e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.440417516901904,
+      "tokens_seen": 2134441984
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.13448884609212e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4404002707733703,
+      "tokens_seen": 2134573056
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.133686406676296e-05,
+      "loss": 1.2963,
+      "theoretical_loss": 3.440383026000287,
+      "tokens_seen": 2134704128
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.132883967260471e-05,
+      "loss": 1.3227,
+      "theoretical_loss": 3.4403657825824636,
+      "tokens_seen": 2134835200
+    },
+    {
+      "epoch": 0.29,
+      "objective/train/docs_used": 717705,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2638752460479736,
+      "objective/train/theoretical_loss": 3.44034854051971,
+      "objective/train/tokens_used": 505425376,
+      "theoretical_loss": 3.44034854051971,
+      "tokens_seen": 2134966272
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.132081527844648e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.44034854051971,
+      "tokens_seen": 2134966272
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.131279088428823e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4403312998118376,
+      "tokens_seen": 2135097344
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.130476649013e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.4403140604586557,
+      "tokens_seen": 2135228416
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.129674209597175e-05,
+      "loss": 1.3254,
+      "theoretical_loss": 3.4402968224599757,
+      "tokens_seen": 2135359488
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.12887177018135e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.4402795858156074,
+      "tokens_seen": 2135490560
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.128069330765527e-05,
+      "loss": 1.3032,
+      "theoretical_loss": 3.4402623505253613,
+      "tokens_seen": 2135621632
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.127266891349703e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.440245116589048,
+      "tokens_seen": 2135752704
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.126464451933879e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.440227884006479,
+      "tokens_seen": 2135883776
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.125662012518055e-05,
+      "loss": 1.2645,
+      "theoretical_loss": 3.4402106527774627,
+      "tokens_seen": 2136014848
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.124859573102231e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4401934229018116,
+      "tokens_seen": 2136145920
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.124057133686407e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.440176194379336,
+      "tokens_seen": 2136276992
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.123254694270582e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.4401589672098463,
+      "tokens_seen": 2136408064
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.122452254854759e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.4401417413931537,
+      "tokens_seen": 2136539136
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 7.121649815438934e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.440124516929068,
+      "tokens_seen": 2136670208
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.12084737602311e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.440107293817401,
+      "tokens_seen": 2136801280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.120044936607286e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.4400900720579632,
+      "tokens_seen": 2136932352
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.119242497191462e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.440072851650566,
+      "tokens_seen": 2137063424
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.118440057775638e-05,
+      "loss": 1.2777,
+      "theoretical_loss": 3.440055632595019,
+      "tokens_seen": 2137194496
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.117637618359813e-05,
+      "loss": 1.1556,
+      "theoretical_loss": 3.440038414891135,
+      "tokens_seen": 2137325568
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.11683517894399e-05,
+      "loss": 1.333,
+      "theoretical_loss": 3.440021198538724,
+      "tokens_seen": 2137456640
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.116032739528165e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.440003983537596,
+      "tokens_seen": 2137587712
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.115230300112342e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.4399867698875646,
+      "tokens_seen": 2137718784
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.114427860696517e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.439969557588439,
+      "tokens_seen": 2137849856
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.113625421280694e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.4399523466400312,
+      "tokens_seen": 2137980928
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.112822981864869e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.439935137042152,
+      "tokens_seen": 2138112000
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.112020542449044e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4399179287946127,
+      "tokens_seen": 2138243072
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.111218103033221e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.439900721897225,
+      "tokens_seen": 2138374144
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.110415663617396e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.4398835163497994,
+      "tokens_seen": 2138505216
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.109613224201573e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.439866312152148,
+      "tokens_seen": 2138636288
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.108810784785748e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.439849109304082,
+      "tokens_seen": 2138767360
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.108008345369925e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.439831907805413,
+      "tokens_seen": 2138898432
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.1072059059541e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.4398147076559518,
+      "tokens_seen": 2139029504
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.106403466538276e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.43979750885551,
+      "tokens_seen": 2139160576
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.105601027122452e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.4397803114039003,
+      "tokens_seen": 2139291648
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.104798587706628e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4397631153009334,
+      "tokens_seen": 2139422720
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.103996148290804e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4397459205464207,
+      "tokens_seen": 2139553792
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.10319370887498e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.4397287271401744,
+      "tokens_seen": 2139684864
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.102391269459156e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.4397115350820053,
+      "tokens_seen": 2139815936
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.101588830043332e-05,
+      "loss": 1.3085,
+      "theoretical_loss": 3.439694344371727,
+      "tokens_seen": 2139947008
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.100786390627508e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.439677155009149,
+      "tokens_seen": 2140078080
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.099983951211684e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.4396599669940846,
+      "tokens_seen": 2140209152
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.099181511795859e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.439642780326346,
+      "tokens_seen": 2140340224
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.098379072380036e-05,
+      "loss": 1.3116,
+      "theoretical_loss": 3.439625595005743,
+      "tokens_seen": 2140471296
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.097576632964211e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.4396084110320895,
+      "tokens_seen": 2140602368
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.096774193548388e-05,
+      "loss": 1.3121,
+      "theoretical_loss": 3.439591228405197,
+      "tokens_seen": 2140733440
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.095971754132563e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.439574047124877,
+      "tokens_seen": 2140864512
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.09516931471674e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.4395568671909422,
+      "tokens_seen": 2140995584
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.094366875300915e-05,
+      "loss": 1.3414,
+      "theoretical_loss": 3.439539688603204,
+      "tokens_seen": 2141126656
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.09356443588509e-05,
+      "loss": 1.234,
+      "theoretical_loss": 3.439522511361475,
+      "tokens_seen": 2141257728
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.092761996469267e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.4395053354655674,
+      "tokens_seen": 2141388800
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 719922,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.307223916053772,
+      "objective/train/theoretical_loss": 3.4394881609152925,
+      "objective/train/tokens_used": 511978976,
+      "theoretical_loss": 3.4394881609152925,
+      "tokens_seen": 2141519872
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.091959557053442e-05,
+      "loss": 1.2812,
+      "theoretical_loss": 3.4394881609152925,
+      "tokens_seen": 2141519872
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.091157117637619e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4394709877104637,
+      "tokens_seen": 2141650944
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.090354678221794e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.439453815850893,
+      "tokens_seen": 2141782016
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.089552238805971e-05,
+      "loss": 1.3524,
+      "theoretical_loss": 3.4394366453363925,
+      "tokens_seen": 2141913088
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.088749799390146e-05,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4394194761667745,
+      "tokens_seen": 2142044160
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.087947359974321e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.439402308341851,
+      "tokens_seen": 2142175232
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.087144920558498e-05,
+      "loss": 1.2856,
+      "theoretical_loss": 3.439385141861435,
+      "tokens_seen": 2142306304
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.086342481142673e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4393679767253387,
+      "tokens_seen": 2142437376
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.08554004172685e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.439350812933375,
+      "tokens_seen": 2142568448
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.084737602311025e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4393336504853558,
+      "tokens_seen": 2142699520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.083935162895202e-05,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4393164893810937,
+      "tokens_seen": 2142830592
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.083132723479377e-05,
+      "loss": 1.3012,
+      "theoretical_loss": 3.439299329620402,
+      "tokens_seen": 2142961664
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.082330284063553e-05,
+      "loss": 1.2947,
+      "theoretical_loss": 3.439282171203093,
+      "tokens_seen": 2143092736
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.081527844647729e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.439265014128979,
+      "tokens_seen": 2143223808
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.080725405231905e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.439247858397873,
+      "tokens_seen": 2143354880
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.079922965816081e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.4392307040095877,
+      "tokens_seen": 2143485952
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.079120526400257e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.4392135509639363,
+      "tokens_seen": 2143617024
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.078318086984433e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.439196399260731,
+      "tokens_seen": 2143748096
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.077515647568609e-05,
+      "loss": 1.2812,
+      "theoretical_loss": 3.4391792488997854,
+      "tokens_seen": 2143879168
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.076713208152784e-05,
+      "loss": 1.3111,
+      "theoretical_loss": 3.439162099880911,
+      "tokens_seen": 2144010240
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.07591076873696e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.4391449522039226,
+      "tokens_seen": 2144141312
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.075108329321136e-05,
+      "loss": 1.2127,
+      "theoretical_loss": 3.4391278058686323,
+      "tokens_seen": 2144272384
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.074305889905313e-05,
+      "loss": 1.3091,
+      "theoretical_loss": 3.4391106608748525,
+      "tokens_seen": 2144403456
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.073503450489488e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.439093517222397,
+      "tokens_seen": 2144534528
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.072701011073665e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.4390763749110786,
+      "tokens_seen": 2144665600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.07189857165784e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4390592339407107,
+      "tokens_seen": 2144796672
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.071096132242017e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.4390420943111066,
+      "tokens_seen": 2144927744
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.070293692826192e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.439024956022079,
+      "tokens_seen": 2145058816
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.069491253410367e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.439007819073441,
+      "tokens_seen": 2145189888
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.068688813994544e-05,
+      "loss": 1.287,
+      "theoretical_loss": 3.438990683465007,
+      "tokens_seen": 2145320960
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.067886374578719e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4389735491965894,
+      "tokens_seen": 2145452032
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.067083935162896e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4389564162680015,
+      "tokens_seen": 2145583104
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.066281495747071e-05,
+      "loss": 1.3398,
+      "theoretical_loss": 3.4389392846790567,
+      "tokens_seen": 2145714176
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.065479056331248e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.4389221544295694,
+      "tokens_seen": 2145845248
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.064676616915423e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.438905025519351,
+      "tokens_seen": 2145976320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.063874177499598e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4388878979482174,
+      "tokens_seen": 2146107392
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.063071738083775e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.438870771715981,
+      "tokens_seen": 2146238464
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.06226929866795e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.438853646822455,
+      "tokens_seen": 2146369536
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.061466859252127e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.4388365232674536,
+      "tokens_seen": 2146500608
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.060664419836302e-05,
+      "loss": 1.313,
+      "theoretical_loss": 3.4388194010507904,
+      "tokens_seen": 2146631680
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.059861980420479e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.438802280172279,
+      "tokens_seen": 2146762752
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.059059541004654e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.438785160631733,
+      "tokens_seen": 2146893824
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.05825710158883e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4387680424289666,
+      "tokens_seen": 2147024896
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.057454662173006e-05,
+      "loss": 1.2679,
+      "theoretical_loss": 3.438750925563793,
+      "tokens_seen": 2147155968
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.056652222757182e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.438733810036026,
+      "tokens_seen": 2147287040
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.055849783341358e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.4387166958454802,
+      "tokens_seen": 2147418112
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.055047343925534e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.438699582991969,
+      "tokens_seen": 2147549184
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.05424490450971e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.438682471475307,
+      "tokens_seen": 2147680256
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.053442465093886e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.438665361295307,
+      "tokens_seen": 2147811328
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.052640025678061e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.4386482524517836,
+      "tokens_seen": 2147942400
+    },
+    {
+      "epoch": 0.3,
+      "objective/train/docs_used": 721999,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3639802932739258,
+      "objective/train/theoretical_loss": 3.4386311449445515,
+      "objective/train/tokens_used": 518532576,
+      "theoretical_loss": 3.4386311449445515,
+      "tokens_seen": 2148073472
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.051837586262238e-05,
+      "loss": 1.3155,
+      "theoretical_loss": 3.4386311449445515,
+      "tokens_seen": 2148073472
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.051035146846413e-05,
+      "loss": 1.1663,
+      "theoretical_loss": 3.438614038773424,
+      "tokens_seen": 2148204544
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.05023270743059e-05,
+      "loss": 1.1464,
+      "theoretical_loss": 3.438596933938215,
+      "tokens_seen": 2148335616
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.049430268014765e-05,
+      "loss": 1.2656,
+      "theoretical_loss": 3.43857983043874,
+      "tokens_seen": 2148466688
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.048627828598942e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.4385627282748112,
+      "tokens_seen": 2148597760
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.047825389183117e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.4385456274462447,
+      "tokens_seen": 2148728832
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.047022949767292e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.4385285279528546,
+      "tokens_seen": 2148859904
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.046220510351469e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.4385114297944543,
+      "tokens_seen": 2148990976
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.045418070935644e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.4384943329708584,
+      "tokens_seen": 2149122048
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.044615631519821e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4384772374818815,
+      "tokens_seen": 2149253120
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.043813192103996e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.438460143327338,
+      "tokens_seen": 2149384192
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.043010752688173e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.4384430505070425,
+      "tokens_seen": 2149515264
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.042208313272348e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.4384259590208095,
+      "tokens_seen": 2149646336
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.041405873856523e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.4384088688684535,
+      "tokens_seen": 2149777408
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.0406034344407e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.4383917800497885,
+      "tokens_seen": 2149908480
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.039800995024875e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.43837469256463,
+      "tokens_seen": 2150039552
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.038998555609052e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4383576064127923,
+      "tokens_seen": 2150170624
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.038196116193227e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.43834052159409,
+      "tokens_seen": 2150301696
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.037393676777404e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.4383234381083376,
+      "tokens_seen": 2150432768
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.03659123736158e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.438306355955351,
+      "tokens_seen": 2150563840
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.035788797945756e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.4382892751349434,
+      "tokens_seen": 2150694912
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.034986358529931e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.438272195646931,
+      "tokens_seen": 2150825984
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.034183919114107e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4382551174911278,
+      "tokens_seen": 2150957056
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.033381479698283e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.4382380406673496,
+      "tokens_seen": 2151088128
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.032579040282459e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.43822096517541,
+      "tokens_seen": 2151219200
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.031776600866635e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4382038910151254,
+      "tokens_seen": 2151350272
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.03097416145081e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.43818681818631,
+      "tokens_seen": 2151481344
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.030171722034987e-05,
+      "loss": 1.2924,
+      "theoretical_loss": 3.4381697466887786,
+      "tokens_seen": 2151612416
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.029369282619163e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.438152676522347,
+      "tokens_seen": 2151743488
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.028566843203338e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.4381356076868306,
+      "tokens_seen": 2151874560
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.027764403787515e-05,
+      "loss": 1.2184,
+      "theoretical_loss": 3.4381185401820438,
+      "tokens_seen": 2152005632
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.02696196437169e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.438101474007802,
+      "tokens_seen": 2152136704
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.026159524955867e-05,
+      "loss": 1.3116,
+      "theoretical_loss": 3.4380844091639204,
+      "tokens_seen": 2152267776
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.025357085540042e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.4380673456502144,
+      "tokens_seen": 2152398848
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.024554646124219e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.4380502834664997,
+      "tokens_seen": 2152529920
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.023752206708394e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4380332226125914,
+      "tokens_seen": 2152660992
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.022949767292569e-05,
+      "loss": 1.2643,
+      "theoretical_loss": 3.4380161630883044,
+      "tokens_seen": 2152792064
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.022147327876746e-05,
+      "loss": 1.3067,
+      "theoretical_loss": 3.4379991048934544,
+      "tokens_seen": 2152923136
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.021344888460921e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4379820480278576,
+      "tokens_seen": 2153054208
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 7.020542449045098e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.4379649924913283,
+      "tokens_seen": 2153185280
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.019740009629273e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.437947938283683,
+      "tokens_seen": 2153316352
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.01893757021345e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.437930885404737,
+      "tokens_seen": 2153447424
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.018135130797625e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4379138338543056,
+      "tokens_seen": 2153578496
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.0173326913818e-05,
+      "loss": 1.3481,
+      "theoretical_loss": 3.437896783632205,
+      "tokens_seen": 2153709568
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.016530251965977e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.4378797347382504,
+      "tokens_seen": 2153840640
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.015727812550152e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.4378626871722577,
+      "tokens_seen": 2153971712
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.014925373134329e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.437845640934043,
+      "tokens_seen": 2154102784
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.014122933718504e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4378285960234214,
+      "tokens_seen": 2154233856
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.013320494302681e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.4378115524402095,
+      "tokens_seen": 2154364928
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.012518054886856e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.4377945101842227,
+      "tokens_seen": 2154496000
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 724229,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2141423225402832,
+      "objective/train/theoretical_loss": 3.437777469255277,
+      "objective/train/tokens_used": 525086176,
+      "theoretical_loss": 3.437777469255277,
+      "tokens_seen": 2154627072
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.011715615471032e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.437777469255277,
+      "tokens_seen": 2154627072
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.010913176055208e-05,
+      "loss": 1.3078,
+      "theoretical_loss": 3.4377604296531885,
+      "tokens_seen": 2154758144
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.010110736639384e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4377433913777726,
+      "tokens_seen": 2154889216
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.00930829722356e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.437726354428846,
+      "tokens_seen": 2155020288
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.008505857807736e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.437709318806225,
+      "tokens_seen": 2155151360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.007703418391912e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4376922845097244,
+      "tokens_seen": 2155282432
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.006900978976088e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.4376752515391615,
+      "tokens_seen": 2155413504
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.006098539560264e-05,
+      "loss": 1.3145,
+      "theoretical_loss": 3.4376582198943524,
+      "tokens_seen": 2155544576
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.00529610014444e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.437641189575112,
+      "tokens_seen": 2155675648
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.004493660728615e-05,
+      "loss": 1.2067,
+      "theoretical_loss": 3.437624160581259,
+      "tokens_seen": 2155806720
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.003691221312792e-05,
+      "loss": 1.3035,
+      "theoretical_loss": 3.437607132912608,
+      "tokens_seen": 2155937792
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.002888781896967e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4375901065689747,
+      "tokens_seen": 2156068864
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.002086342481144e-05,
+      "loss": 1.2979,
+      "theoretical_loss": 3.437573081550177,
+      "tokens_seen": 2156199936
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.001283903065319e-05,
+      "loss": 1.3267,
+      "theoretical_loss": 3.43755605785603,
+      "tokens_seen": 2156331008
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 7.000481463649496e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4375390354863513,
+      "tokens_seen": 2156462080
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.999679024233671e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.4375220144409564,
+      "tokens_seen": 2156593152
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.998876584817846e-05,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4375049947196628,
+      "tokens_seen": 2156724224
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.998074145402023e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.437487976322286,
+      "tokens_seen": 2156855296
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.997271705986198e-05,
+      "loss": 1.3118,
+      "theoretical_loss": 3.437470959248643,
+      "tokens_seen": 2156986368
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.996469266570375e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.4374539434985505,
+      "tokens_seen": 2157117440
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.99566682715455e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.4374369290718247,
+      "tokens_seen": 2157248512
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.994864387738727e-05,
+      "loss": 1.3071,
+      "theoretical_loss": 3.437419915968283,
+      "tokens_seen": 2157379584
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.994061948322902e-05,
+      "loss": 1.3351,
+      "theoretical_loss": 3.437402904187742,
+      "tokens_seen": 2157510656
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.993259508907077e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4373858937300175,
+      "tokens_seen": 2157641728
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.992457069491254e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.4373688845949273,
+      "tokens_seen": 2157772800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.99165463007543e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.437351876782288,
+      "tokens_seen": 2157903872
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.990852190659606e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.4373348702919166,
+      "tokens_seen": 2158034944
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.990049751243781e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.4373178651236294,
+      "tokens_seen": 2158166016
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.989247311827958e-05,
+      "loss": 1.326,
+      "theoretical_loss": 3.4373008612772438,
+      "tokens_seen": 2158297088
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.988444872412133e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.4372838587525765,
+      "tokens_seen": 2158428160
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.987642432996309e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.437266857549445,
+      "tokens_seen": 2158559232
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.986839993580485e-05,
+      "loss": 1.2777,
+      "theoretical_loss": 3.437249857667666,
+      "tokens_seen": 2158690304
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.98603755416466e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.4372328591070564,
+      "tokens_seen": 2158821376
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.985235114748837e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.437215861867434,
+      "tokens_seen": 2158952448
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.984432675333013e-05,
+      "loss": 1.3209,
+      "theoretical_loss": 3.437198865948615,
+      "tokens_seen": 2159083520
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.98363023591719e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.4371818713504174,
+      "tokens_seen": 2159214592
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.982827796501365e-05,
+      "loss": 1.3681,
+      "theoretical_loss": 3.4371648780726582,
+      "tokens_seen": 2159345664
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.98202535708554e-05,
+      "loss": 1.3429,
+      "theoretical_loss": 3.4371478861151545,
+      "tokens_seen": 2159476736
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.981222917669717e-05,
+      "loss": 1.3239,
+      "theoretical_loss": 3.4371308954777233,
+      "tokens_seen": 2159607808
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.980420478253892e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.4371139061601825,
+      "tokens_seen": 2159738880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.979618038838069e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.4370969181623496,
+      "tokens_seen": 2159869952
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.978815599422244e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4370799314840417,
+      "tokens_seen": 2160001024
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.97801316000642e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.437062946125076,
+      "tokens_seen": 2160132096
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.977210720590596e-05,
+      "loss": 1.2406,
+      "theoretical_loss": 3.4370459620852705,
+      "tokens_seen": 2160263168
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.976408281174773e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.437028979364442,
+      "tokens_seen": 2160394240
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.975605841758948e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4370119979624087,
+      "tokens_seen": 2160525312
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.974803402343123e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.436995017878988,
+      "tokens_seen": 2160656384
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.9740009629273e-05,
+      "loss": 1.3955,
+      "theoretical_loss": 3.4369780391139977,
+      "tokens_seen": 2160787456
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.973198523511475e-05,
+      "loss": 1.3109,
+      "theoretical_loss": 3.436961061667255,
+      "tokens_seen": 2160918528
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.972396084095652e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.436944085538578,
+      "tokens_seen": 2161049600
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 726764,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2552740573883057,
+      "objective/train/theoretical_loss": 3.436927110727784,
+      "objective/train/tokens_used": 531639776,
+      "theoretical_loss": 3.436927110727784,
+      "tokens_seen": 2161180672
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.971593644679827e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.436927110727784,
+      "tokens_seen": 2161180672
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.970791205264004e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.436910137234692,
+      "tokens_seen": 2161311744
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.969988765848179e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4368931650591183,
+      "tokens_seen": 2161442816
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.969186326432354e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4368761942008814,
+      "tokens_seen": 2161573888
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.968383887016531e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.436859224659799,
+      "tokens_seen": 2161704960
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.967581447600706e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.436842256435689,
+      "tokens_seen": 2161836032
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.966779008184883e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.43682528952837,
+      "tokens_seen": 2161967104
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.965976568769058e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.4368083239376594,
+      "tokens_seen": 2162098176
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.965174129353235e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.436791359663375,
+      "tokens_seen": 2162229248
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.96437168993741e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.4367743967053355,
+      "tokens_seen": 2162360320
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.963569250521586e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.4367574350633587,
+      "tokens_seen": 2162491392
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.962766811105762e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4367404747372623,
+      "tokens_seen": 2162622464
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.961964371689938e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4367235157268654,
+      "tokens_seen": 2162753536
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.961161932274114e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.436706558031985,
+      "tokens_seen": 2162884608
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.96035949285829e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4366896016524406,
+      "tokens_seen": 2163015680
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.959557053442466e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.43667264658805,
+      "tokens_seen": 2163146752
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.958754614026642e-05,
+      "loss": 1.3158,
+      "theoretical_loss": 3.436655692838631,
+      "tokens_seen": 2163277824
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.957952174610817e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.436638740404003,
+      "tokens_seen": 2163408896
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.957149735194994e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.436621789283983,
+      "tokens_seen": 2163539968
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.956347295779169e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.436604839478391,
+      "tokens_seen": 2163671040
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.955544856363346e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.4365878909870435,
+      "tokens_seen": 2163802112
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.954742416947521e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.436570943809761,
+      "tokens_seen": 2163933184
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.953939977531698e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.436553997946361,
+      "tokens_seen": 2164064256
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.953137538115873e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.436537053396662,
+      "tokens_seen": 2164195328
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.952335098700048e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.436520110160483,
+      "tokens_seen": 2164326400
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.951532659284225e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.436503168237643,
+      "tokens_seen": 2164457472
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.9507302198684e-05,
+      "loss": 1.2406,
+      "theoretical_loss": 3.4364862276279595,
+      "tokens_seen": 2164588544
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.949927780452577e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.436469288331252,
+      "tokens_seen": 2164719616
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.949125341036752e-05,
+      "loss": 1.3329,
+      "theoretical_loss": 3.4364523503473396,
+      "tokens_seen": 2164850688
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.948322901620929e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.4364354136760404,
+      "tokens_seen": 2164981760
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.947520462205104e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.436418478317173,
+      "tokens_seen": 2165112832
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.94671802278928e-05,
+      "loss": 1.3167,
+      "theoretical_loss": 3.436401544270557,
+      "tokens_seen": 2165243904
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.945915583373456e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.436384611536011,
+      "tokens_seen": 2165374976
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.945113143957631e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.436367680113354,
+      "tokens_seen": 2165506048
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.944310704541808e-05,
+      "loss": 1.3116,
+      "theoretical_loss": 3.4363507500024046,
+      "tokens_seen": 2165637120
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.943508265125983e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.436333821202982,
+      "tokens_seen": 2165768192
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.94270582571016e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.4363168937149053,
+      "tokens_seen": 2165899264
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.941903386294335e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.4362999675379937,
+      "tokens_seen": 2166030336
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.941100946878512e-05,
+      "loss": 1.3142,
+      "theoretical_loss": 3.4362830426720663,
+      "tokens_seen": 2166161408
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.940298507462687e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.4362661191169424,
+      "tokens_seen": 2166292480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.939496068046863e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.4362491968724402,
+      "tokens_seen": 2166423552
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.93869362863104e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.43623227593838,
+      "tokens_seen": 2166554624
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.937891189215215e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.4362153563145807,
+      "tokens_seen": 2166685696
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.937088749799391e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.4361984380008614,
+      "tokens_seen": 2166816768
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.936286310383567e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.436181520997042,
+      "tokens_seen": 2166947840
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.935483870967743e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.436164605302941,
+      "tokens_seen": 2167078912
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.934681431551919e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.4361476909183786,
+      "tokens_seen": 2167209984
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.933878992136094e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4361307778431733,
+      "tokens_seen": 2167341056
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.93307655272027e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4361138660771458,
+      "tokens_seen": 2167472128
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.932274113304446e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.4360969556201146,
+      "tokens_seen": 2167603200
+    },
+    {
+      "epoch": 0.31,
+      "objective/train/docs_used": 728956,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.238715410232544,
+      "objective/train/theoretical_loss": 3.4360800464718992,
+      "objective/train/tokens_used": 538193376,
+      "theoretical_loss": 3.4360800464718992,
+      "tokens_seen": 2167734272
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.931471673888623e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.4360800464718992,
+      "tokens_seen": 2167734272
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.930669234472798e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4360631386323197,
+      "tokens_seen": 2167865344
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.929866795056975e-05,
+      "loss": 1.2961,
+      "theoretical_loss": 3.4360462321011958,
+      "tokens_seen": 2167996416
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.92906435564115e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.436029326878347,
+      "tokens_seen": 2168127488
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.928261916225325e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.436012422963593,
+      "tokens_seen": 2168258560
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.927459476809502e-05,
+      "loss": 1.3096,
+      "theoretical_loss": 3.4359955203567525,
+      "tokens_seen": 2168389632
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.926657037393677e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.435978619057647,
+      "tokens_seen": 2168520704
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.925854597977854e-05,
+      "loss": 1.2768,
+      "theoretical_loss": 3.4359617190660954,
+      "tokens_seen": 2168651776
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.925052158562029e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.435944820381917,
+      "tokens_seen": 2168782848
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.924249719146206e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.4359279230049324,
+      "tokens_seen": 2168913920
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.923447279730381e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4359110269349618,
+      "tokens_seen": 2169044992
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.922644840314556e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.435894132171824,
+      "tokens_seen": 2169176064
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.921842400898733e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.43587723871534,
+      "tokens_seen": 2169307136
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.921039961482908e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.4358603465653292,
+      "tokens_seen": 2169438208
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.920237522067085e-05,
+      "loss": 1.3121,
+      "theoretical_loss": 3.435843455721612,
+      "tokens_seen": 2169569280
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.91943508265126e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.4358265661840086,
+      "tokens_seen": 2169700352
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.918632643235437e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.435809677952338,
+      "tokens_seen": 2169831424
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.917830203819612e-05,
+      "loss": 1.3076,
+      "theoretical_loss": 3.435792791026422,
+      "tokens_seen": 2169962496
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.917027764403788e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.4357759054060795,
+      "tokens_seen": 2170093568
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.916225324987964e-05,
+      "loss": 1.2997,
+      "theoretical_loss": 3.4357590210911315,
+      "tokens_seen": 2170224640
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.91542288557214e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.4357421380813977,
+      "tokens_seen": 2170355712
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.914620446156316e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4357252563766987,
+      "tokens_seen": 2170486784
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.913818006740492e-05,
+      "loss": 1.2628,
+      "theoretical_loss": 3.4357083759768545,
+      "tokens_seen": 2170617856
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.913015567324668e-05,
+      "loss": 1.1823,
+      "theoretical_loss": 3.435691496881686,
+      "tokens_seen": 2170748928
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.912213127908844e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.4356746190910132,
+      "tokens_seen": 2170880000
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.911410688493019e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.4356577426046564,
+      "tokens_seen": 2171011072
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.910608249077196e-05,
+      "loss": 1.343,
+      "theoretical_loss": 3.4356408674224364,
+      "tokens_seen": 2171142144
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.909805809661371e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.4356239935441737,
+      "tokens_seen": 2171273216
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.909003370245548e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.4356071209696886,
+      "tokens_seen": 2171404288
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.908200930829723e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.4355902496988016,
+      "tokens_seen": 2171535360
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.907398491413898e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.435573379731334,
+      "tokens_seen": 2171666432
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.906596051998075e-05,
+      "loss": 1.2904,
+      "theoretical_loss": 3.4355565110671056,
+      "tokens_seen": 2171797504
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.90579361258225e-05,
+      "loss": 1.3269,
+      "theoretical_loss": 3.435539643705938,
+      "tokens_seen": 2171928576
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.904991173166427e-05,
+      "loss": 1.2908,
+      "theoretical_loss": 3.4355227776476505,
+      "tokens_seen": 2172059648
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.904188733750602e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.435505912892065,
+      "tokens_seen": 2172190720
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.903386294334779e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.435489049439002,
+      "tokens_seen": 2172321792
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.902583854918954e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.435472187288282,
+      "tokens_seen": 2172452864
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.90178141550313e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4354553264397265,
+      "tokens_seen": 2172583936
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.900978976087306e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.4354384668931557,
+      "tokens_seen": 2172715008
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.900176536671482e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.435421608648391,
+      "tokens_seen": 2172846080
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.899374097255658e-05,
+      "loss": 1.3411,
+      "theoretical_loss": 3.4354047517052537,
+      "tokens_seen": 2172977152
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.898571657839833e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.4353878960635638,
+      "tokens_seen": 2173108224
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.897769218424009e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.4353710417231427,
+      "tokens_seen": 2173239296
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.896966779008185e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.435354188683812,
+      "tokens_seen": 2173370368
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.896164339592361e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.435337336945392,
+      "tokens_seen": 2173501440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.895361900176537e-05,
+      "loss": 1.2252,
+      "theoretical_loss": 3.435320486507705,
+      "tokens_seen": 2173632512
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.894559460760713e-05,
+      "loss": 1.291,
+      "theoretical_loss": 3.4353036373705708,
+      "tokens_seen": 2173763584
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.89375702134489e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4352867895338113,
+      "tokens_seen": 2173894656
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.892954581929065e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.435269942997248,
+      "tokens_seen": 2174025728
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.89215214251324e-05,
+      "loss": 1.296,
+      "theoretical_loss": 3.4352530977607016,
+      "tokens_seen": 2174156800
+    },
+    {
+      "epoch": 0.32,
+      "objective/train/docs_used": 731141,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2738087177276611,
+      "objective/train/theoretical_loss": 3.435236253823994,
+      "objective/train/tokens_used": 544746976,
+      "theoretical_loss": 3.435236253823994,
+      "tokens_seen": 2174287872
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.891349703097417e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.435236253823994,
+      "tokens_seen": 2174287872
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.890547263681592e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.435219411186946,
+      "tokens_seen": 2174418944
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.889744824265769e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.435202569849379,
+      "tokens_seen": 2174550016
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.888942384849944e-05,
+      "loss": 1.2921,
+      "theoretical_loss": 3.4351857298111153,
+      "tokens_seen": 2174681088
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.88813994543412e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4351688910719753,
+      "tokens_seen": 2174812160
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.887337506018296e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.4351520536317808,
+      "tokens_seen": 2174943232
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.886535066602471e-05,
+      "loss": 1.3206,
+      "theoretical_loss": 3.4351352174903536,
+      "tokens_seen": 2175074304
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.885732627186648e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.4351183826475147,
+      "tokens_seen": 2175205376
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.884930187770823e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.4351015491030865,
+      "tokens_seen": 2175336448
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.884127748355e-05,
+      "loss": 1.335,
+      "theoretical_loss": 3.43508471685689,
+      "tokens_seen": 2175467520
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.883325308939175e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4350678859087473,
+      "tokens_seen": 2175598592
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.88252286952335e-05,
+      "loss": 1.2438,
+      "theoretical_loss": 3.4350510562584797,
+      "tokens_seen": 2175729664
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.881720430107527e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.435034227905909,
+      "tokens_seen": 2175860736
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.880917990691703e-05,
+      "loss": 1.2959,
+      "theoretical_loss": 3.4350174008508576,
+      "tokens_seen": 2175991808
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.880115551275879e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.4350005750931465,
+      "tokens_seen": 2176122880
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.879313111860055e-05,
+      "loss": 1.2869,
+      "theoretical_loss": 3.4349837506325978,
+      "tokens_seen": 2176253952
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.87851067244423e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.4349669274690338,
+      "tokens_seen": 2176385024
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.877708233028407e-05,
+      "loss": 1.3062,
+      "theoretical_loss": 3.4349501056022755,
+      "tokens_seen": 2176516096
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.876905793612582e-05,
+      "loss": 1.291,
+      "theoretical_loss": 3.434933285032146,
+      "tokens_seen": 2176647168
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.876103354196759e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.434916465758466,
+      "tokens_seen": 2176778240
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.875300914780934e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.4348996477810587,
+      "tokens_seen": 2176909312
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.87449847536511e-05,
+      "loss": 1.2747,
+      "theoretical_loss": 3.4348828310997455,
+      "tokens_seen": 2177040384
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.873696035949286e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.4348660157143485,
+      "tokens_seen": 2177171456
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.872893596533461e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.4348492016246905,
+      "tokens_seen": 2177302528
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.872091157117638e-05,
+      "loss": 1.3136,
+      "theoretical_loss": 3.4348323888305927,
+      "tokens_seen": 2177433600
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.871288717701813e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.434815577331878,
+      "tokens_seen": 2177564672
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.87048627828599e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.434798767128368,
+      "tokens_seen": 2177695744
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.869683838870165e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.4347819582198857,
+      "tokens_seen": 2177826816
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.86888139945434e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.434765150606253,
+      "tokens_seen": 2177957888
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.868078960038517e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.434748344287292,
+      "tokens_seen": 2178088960
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.867276520622692e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.4347315392628257,
+      "tokens_seen": 2178220032
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.866474081206869e-05,
+      "loss": 1.3214,
+      "theoretical_loss": 3.4347147355326757,
+      "tokens_seen": 2178351104
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.865671641791044e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.434697933096665,
+      "tokens_seen": 2178482176
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.864869202375221e-05,
+      "loss": 1.1557,
+      "theoretical_loss": 3.4346811319546164,
+      "tokens_seen": 2178613248
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.864066762959396e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.434664332106352,
+      "tokens_seen": 2178744320
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.863264323543572e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.4346475335516935,
+      "tokens_seen": 2178875392
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.862461884127748e-05,
+      "loss": 1.172,
+      "theoretical_loss": 3.434630736290465,
+      "tokens_seen": 2179006464
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.861659444711924e-05,
+      "loss": 1.3136,
+      "theoretical_loss": 3.4346139403224885,
+      "tokens_seen": 2179137536
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.8608570052961e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.434597145647586,
+      "tokens_seen": 2179268608
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.860054565880276e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.4345803522655816,
+      "tokens_seen": 2179399680
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.859252126464452e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.4345635601762967,
+      "tokens_seen": 2179530752
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.858449687048628e-05,
+      "loss": 1.3083,
+      "theoretical_loss": 3.4345467693795544,
+      "tokens_seen": 2179661824
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.857647247632803e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.434529979875178,
+      "tokens_seen": 2179792896
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.85684480821698e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.4345131916629894,
+      "tokens_seen": 2179923968
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.856042368801155e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.434496404742813,
+      "tokens_seen": 2180055040
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.855239929385332e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.4344796191144695,
+      "tokens_seen": 2180186112
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.854437489969507e-05,
+      "loss": 1.1913,
+      "theoretical_loss": 3.4344628347777837,
+      "tokens_seen": 2180317184
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.853635050553684e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.434446051732578,
+      "tokens_seen": 2180448256
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.852832611137859e-05,
+      "loss": 1.3398,
+      "theoretical_loss": 3.4344292699786756,
+      "tokens_seen": 2180579328
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.852030171722034e-05,
+      "loss": 1.2604,
+      "theoretical_loss": 3.4344124895158985,
+      "tokens_seen": 2180710400
+    },
+    {
+      "epoch": 0.32,
+      "objective/train/docs_used": 733364,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.231123447418213,
+      "objective/train/theoretical_loss": 3.434395710344071,
+      "objective/train/tokens_used": 551300576,
+      "theoretical_loss": 3.434395710344071,
+      "tokens_seen": 2180841472
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.851227732306211e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.434395710344071,
+      "tokens_seen": 2180841472
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.850425292890386e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.434378932463016,
+      "tokens_seen": 2180972544
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.849622853474563e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.434362155872556,
+      "tokens_seen": 2181103616
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.848820414058738e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.4343453805725144,
+      "tokens_seen": 2181234688
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.848017974642915e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4343286065627145,
+      "tokens_seen": 2181365760
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.84721553522709e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.43431183384298,
+      "tokens_seen": 2181496832
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.846413095811267e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.434295062413134,
+      "tokens_seen": 2181627904
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.845610656395442e-05,
+      "loss": 1.3329,
+      "theoretical_loss": 3.434278292272999,
+      "tokens_seen": 2181758976
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.844808216979617e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.434261523422399,
+      "tokens_seen": 2181890048
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.844005777563794e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4342447558611577,
+      "tokens_seen": 2182021120
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.84320333814797e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.434227989589098,
+      "tokens_seen": 2182152192
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.842400898732146e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.4342112246060434,
+      "tokens_seen": 2182283264
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.841598459316321e-05,
+      "loss": 1.3128,
+      "theoretical_loss": 3.434194460911818,
+      "tokens_seen": 2182414336
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.840796019900498e-05,
+      "loss": 1.2799,
+      "theoretical_loss": 3.434177698506244,
+      "tokens_seen": 2182545408
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.839993580484673e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.4341609373891466,
+      "tokens_seen": 2182676480
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.839191141068849e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.434144177560348,
+      "tokens_seen": 2182807552
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.838388701653025e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.434127419019673,
+      "tokens_seen": 2182938624
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.8375862622372e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.434110661766944,
+      "tokens_seen": 2183069696
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.836783822821377e-05,
+      "loss": 1.2946,
+      "theoretical_loss": 3.4340939058019857,
+      "tokens_seen": 2183200768
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.835981383405553e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4340771511246215,
+      "tokens_seen": 2183331840
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.835178943989729e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.434060397734675,
+      "tokens_seen": 2183462912
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.834376504573905e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4340436456319705,
+      "tokens_seen": 2183593984
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.83357406515808e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.434026894816331,
+      "tokens_seen": 2183725056
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.832771625742257e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4340101452875813,
+      "tokens_seen": 2183856128
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.831969186326432e-05,
+      "loss": 1.3358,
+      "theoretical_loss": 3.433993397045545,
+      "tokens_seen": 2183987200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.831166746910609e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4339766500900453,
+      "tokens_seen": 2184118272
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.830364307494784e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.4339599044209073,
+      "tokens_seen": 2184249344
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.82956186807896e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.433943160037954,
+      "tokens_seen": 2184380416
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.828759428663136e-05,
+      "loss": 1.3016,
+      "theoretical_loss": 3.4339264169410106,
+      "tokens_seen": 2184511488
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.827956989247311e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.4339096751299003,
+      "tokens_seen": 2184642560
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.827154549831488e-05,
+      "loss": 1.2861,
+      "theoretical_loss": 3.4338929346044473,
+      "tokens_seen": 2184773632
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.826352110415663e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.433876195364476,
+      "tokens_seen": 2184904704
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.82554967099984e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.4338594574098105,
+      "tokens_seen": 2185035776
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.824747231584015e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.433842720740275,
+      "tokens_seen": 2185166848
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.823944792168192e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.4338259853556936,
+      "tokens_seen": 2185297920
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.823142352752367e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.433809251255891,
+      "tokens_seen": 2185428992
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.822339913336542e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.4337925184406908,
+      "tokens_seen": 2185560064
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.821537473920719e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.433775786909918,
+      "tokens_seen": 2185691136
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.820735034504894e-05,
+      "loss": 1.2813,
+      "theoretical_loss": 3.433759056663397,
+      "tokens_seen": 2185822208
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.819932595089071e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.433742327700952,
+      "tokens_seen": 2185953280
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.819130155673246e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.4337256000224072,
+      "tokens_seen": 2186084352
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.818327716257423e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4337088736275874,
+      "tokens_seen": 2186215424
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.817525276841598e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.433692148516317,
+      "tokens_seen": 2186346496
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.816722837425775e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.4336754246884214,
+      "tokens_seen": 2186477568
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.81592039800995e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.433658702143724,
+      "tokens_seen": 2186608640
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.815117958594126e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.4336419808820495,
+      "tokens_seen": 2186739712
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.814315519178302e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.4336252609032236,
+      "tokens_seen": 2186870784
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.813513079762478e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.43360854220707,
+      "tokens_seen": 2187001856
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.812710640346654e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.433591824793414,
+      "tokens_seen": 2187132928
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.81190820093083e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.4335751086620796,
+      "tokens_seen": 2187264000
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 735627,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1929986476898193,
+      "objective/train/theoretical_loss": 3.4335583938128926,
+      "objective/train/tokens_used": 557854176,
+      "theoretical_loss": 3.4335583938128926,
+      "tokens_seen": 2187395072
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.811105761515006e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.4335583938128926,
+      "tokens_seen": 2187395072
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.810303322099182e-05,
+      "loss": 1.185,
+      "theoretical_loss": 3.4335416802456775,
+      "tokens_seen": 2187526144
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.809500882683357e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.433524967960259,
+      "tokens_seen": 2187657216
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.808698443267534e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.433508256956462,
+      "tokens_seen": 2187788288
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.807896003851709e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4334915472341114,
+      "tokens_seen": 2187919360
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.807093564435886e-05,
+      "loss": 1.2964,
+      "theoretical_loss": 3.4334748387930327,
+      "tokens_seen": 2188050432
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.806291125020061e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.4334581316330506,
+      "tokens_seen": 2188181504
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.805488685604238e-05,
+      "loss": 1.1754,
+      "theoretical_loss": 3.4334414257539896,
+      "tokens_seen": 2188312576
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.804686246188413e-05,
+      "loss": 1.3253,
+      "theoretical_loss": 3.433424721155675,
+      "tokens_seen": 2188443648
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.803883806772588e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.433408017837933,
+      "tokens_seen": 2188574720
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.803081367356765e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.4333913158005873,
+      "tokens_seen": 2188705792
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.80227892794094e-05,
+      "loss": 1.1621,
+      "theoretical_loss": 3.433374615043464,
+      "tokens_seen": 2188836864
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.801476488525117e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.433357915566388,
+      "tokens_seen": 2188967936
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.800674049109292e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.4333412173691844,
+      "tokens_seen": 2189099008
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.799871609693469e-05,
+      "loss": 1.1486,
+      "theoretical_loss": 3.433324520451679,
+      "tokens_seen": 2189230080
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.799069170277644e-05,
+      "loss": 1.1699,
+      "theoretical_loss": 3.433307824813696,
+      "tokens_seen": 2189361152
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.79826673086182e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.433291130455062,
+      "tokens_seen": 2189492224
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.797464291445996e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.4332744373756015,
+      "tokens_seen": 2189623296
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.796661852030171e-05,
+      "loss": 1.2841,
+      "theoretical_loss": 3.433257745575141,
+      "tokens_seen": 2189754368
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.795859412614348e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.4332410550535046,
+      "tokens_seen": 2189885440
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.795056973198523e-05,
+      "loss": 1.3136,
+      "theoretical_loss": 3.433224365810519,
+      "tokens_seen": 2190016512
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.7942545337827e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.433207677846009,
+      "tokens_seen": 2190147584
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.793452094366875e-05,
+      "loss": 1.3114,
+      "theoretical_loss": 3.4331909911598,
+      "tokens_seen": 2190278656
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.792649654951051e-05,
+      "loss": 1.2815,
+      "theoretical_loss": 3.4331743057517183,
+      "tokens_seen": 2190409728
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.791847215535227e-05,
+      "loss": 1.1954,
+      "theoretical_loss": 3.4331576216215893,
+      "tokens_seen": 2190540800
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.791044776119403e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.4331409387692386,
+      "tokens_seen": 2190671872
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.79024233670358e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4331242571944913,
+      "tokens_seen": 2190802944
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.789439897287755e-05,
+      "loss": 1.3485,
+      "theoretical_loss": 3.433107576897174,
+      "tokens_seen": 2190934016
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.788637457871931e-05,
+      "loss": 1.3264,
+      "theoretical_loss": 3.433090897877112,
+      "tokens_seen": 2191065088
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.787835018456107e-05,
+      "loss": 1.1755,
+      "theoretical_loss": 3.4330742201341318,
+      "tokens_seen": 2191196160
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.787032579040283e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.433057543668058,
+      "tokens_seen": 2191327232
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.786230139624459e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.4330408684787175,
+      "tokens_seen": 2191458304
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.785427700208634e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.4330241945659354,
+      "tokens_seen": 2191589376
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.78462526079281e-05,
+      "loss": 1.3108,
+      "theoretical_loss": 3.4330075219295386,
+      "tokens_seen": 2191720448
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.783822821376986e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.432990850569352,
+      "tokens_seen": 2191851520
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.783020381961163e-05,
+      "loss": 1.2985,
+      "theoretical_loss": 3.4329741804852025,
+      "tokens_seen": 2191982592
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.782217942545338e-05,
+      "loss": 1.3028,
+      "theoretical_loss": 3.432957511676916,
+      "tokens_seen": 2192113664
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.781415503129515e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.432940844144318,
+      "tokens_seen": 2192244736
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.78061306371369e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.432924177887235,
+      "tokens_seen": 2192375808
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.779810624297865e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.432907512905493,
+      "tokens_seen": 2192506880
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.779008184882042e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.4328908491989183,
+      "tokens_seen": 2192637952
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.778205745466217e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.432874186767337,
+      "tokens_seen": 2192769024
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.777403306050394e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4328575256105758,
+      "tokens_seen": 2192900096
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.776600866634569e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.43284086572846,
+      "tokens_seen": 2193031168
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.775798427218746e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4328242071208175,
+      "tokens_seen": 2193162240
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.774995987802921e-05,
+      "loss": 1.2651,
+      "theoretical_loss": 3.432807549787473,
+      "tokens_seen": 2193293312
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.774193548387096e-05,
+      "loss": 1.287,
+      "theoretical_loss": 3.4327908937282534,
+      "tokens_seen": 2193424384
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.773391108971273e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.432774238942985,
+      "tokens_seen": 2193555456
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.772588669555448e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4327575854314953,
+      "tokens_seen": 2193686528
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.771786230139625e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.4327409331936094,
+      "tokens_seen": 2193817600
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 737824,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2871661186218262,
+      "objective/train/theoretical_loss": 3.4327242822291546,
+      "objective/train/tokens_used": 564407776,
+      "theoretical_loss": 3.4327242822291546,
+      "tokens_seen": 2193948672
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.7709837907238e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.4327242822291546,
+      "tokens_seen": 2193948672
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.770181351307977e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.4327076325379573,
+      "tokens_seen": 2194079744
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.769378911892152e-05,
+      "loss": 1.1922,
+      "theoretical_loss": 3.432690984119844,
+      "tokens_seen": 2194210816
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.768576472476328e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.432674336974641,
+      "tokens_seen": 2194341888
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.767774033060504e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.432657691102176,
+      "tokens_seen": 2194472960
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.76697159364468e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.4326410465022743,
+      "tokens_seen": 2194604032
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.766169154228856e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.4326244031747635,
+      "tokens_seen": 2194735104
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.765366714813032e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.43260776111947,
+      "tokens_seen": 2194866176
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.764564275397208e-05,
+      "loss": 1.2917,
+      "theoretical_loss": 3.432591120336221,
+      "tokens_seen": 2194997248
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.763761835981384e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.432574480824843,
+      "tokens_seen": 2195128320
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.762959396565559e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.4325578425851626,
+      "tokens_seen": 2195259392
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.762156957149736e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.4325412056170075,
+      "tokens_seen": 2195390464
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.761354517733911e-05,
+      "loss": 1.2679,
+      "theoretical_loss": 3.432524569920204,
+      "tokens_seen": 2195521536
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.760552078318088e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.432507935494579,
+      "tokens_seen": 2195652608
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.759749638902263e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.4324913023399595,
+      "tokens_seen": 2195783680
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.75894719948644e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.432474670456173,
+      "tokens_seen": 2195914752
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.758144760070615e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.432458039843046,
+      "tokens_seen": 2196045824
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.75734232065479e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.432441410500406,
+      "tokens_seen": 2196176896
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.756539881238967e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.43242478242808,
+      "tokens_seen": 2196307968
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.755737441823142e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.432408155625895,
+      "tokens_seen": 2196439040
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.754935002407319e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4323915300936783,
+      "tokens_seen": 2196570112
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.754132562991494e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4323749058312574,
+      "tokens_seen": 2196701184
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.753330123575671e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.432358282838459,
+      "tokens_seen": 2196832256
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.752527684159846e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4323416611151107,
+      "tokens_seen": 2196963328
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.751725244744023e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.4323250406610404,
+      "tokens_seen": 2197094400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.750922805328198e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.4323084214760744,
+      "tokens_seen": 2197225472
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.750120365912373e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4322918035600405,
+      "tokens_seen": 2197356544
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.74931792649655e-05,
+      "loss": 1.3129,
+      "theoretical_loss": 3.4322751869127663,
+      "tokens_seen": 2197487616
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.748515487080725e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.432258571534079,
+      "tokens_seen": 2197618688
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.747713047664902e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4322419574238068,
+      "tokens_seen": 2197749760
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.746910608249077e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4322253445817763,
+      "tokens_seen": 2197880832
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.746108168833254e-05,
+      "loss": 1.3266,
+      "theoretical_loss": 3.4322087330078155,
+      "tokens_seen": 2198011904
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.74530572941743e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.432192122701752,
+      "tokens_seen": 2198142976
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.744503290001605e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.432175513663413,
+      "tokens_seen": 2198274048
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.743700850585781e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4321589058926274,
+      "tokens_seen": 2198405120
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.742898411169957e-05,
+      "loss": 1.303,
+      "theoretical_loss": 3.4321422993892217,
+      "tokens_seen": 2198536192
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.742095971754133e-05,
+      "loss": 1.3292,
+      "theoretical_loss": 3.432125694153024,
+      "tokens_seen": 2198667264
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.741293532338309e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.432109090183862,
+      "tokens_seen": 2198798336
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.740491092922485e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4320924874815635,
+      "tokens_seen": 2198929408
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.73968865350666e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.4320758860459564,
+      "tokens_seen": 2199060480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.738886214090836e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.4320592858768686,
+      "tokens_seen": 2199191552
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.738083774675013e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.432042686974128,
+      "tokens_seen": 2199322624
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.737281335259188e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.4320260893375623,
+      "tokens_seen": 2199453696
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.736478895843365e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.432009492967,
+      "tokens_seen": 2199584768
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.73567645642754e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4319928978622687,
+      "tokens_seen": 2199715840
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.734874017011717e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.4319763040231965,
+      "tokens_seen": 2199846912
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.734071577595892e-05,
+      "loss": 1.3013,
+      "theoretical_loss": 3.431959711449611,
+      "tokens_seen": 2199977984
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.733269138180067e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.4319431201413417,
+      "tokens_seen": 2200109056
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.732466698764244e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.4319265300982154,
+      "tokens_seen": 2200240128
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.731664259348419e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.4319099413200607,
+      "tokens_seen": 2200371200
+    },
+    {
+      "epoch": 0.33,
+      "objective/train/docs_used": 740003,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1829400062561035,
+      "objective/train/theoretical_loss": 3.431893353806706,
+      "objective/train/tokens_used": 570961376,
+      "theoretical_loss": 3.431893353806706,
+      "tokens_seen": 2200502272
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.730861819932596e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.431893353806706,
+      "tokens_seen": 2200502272
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.730059380516771e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.431876767557979,
+      "tokens_seen": 2200633344
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.729256941100948e-05,
+      "loss": 1.3511,
+      "theoretical_loss": 3.431860182573708,
+      "tokens_seen": 2200764416
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.728454501685123e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.431843598853722,
+      "tokens_seen": 2200895488
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.727652062269298e-05,
+      "loss": 1.1739,
+      "theoretical_loss": 3.431827016397849,
+      "tokens_seen": 2201026560
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.726849622853475e-05,
+      "loss": 1.3199,
+      "theoretical_loss": 3.431810435205917,
+      "tokens_seen": 2201157632
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.72604718343765e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.431793855277755,
+      "tokens_seen": 2201288704
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.725244744021827e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.4317772766131913,
+      "tokens_seen": 2201419776
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.724442304606002e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.431760699212054,
+      "tokens_seen": 2201550848
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.723639865190179e-05,
+      "loss": 1.295,
+      "theoretical_loss": 3.4317441230741723,
+      "tokens_seen": 2201681920
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.722837425774354e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.431727548199374,
+      "tokens_seen": 2201812992
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.722034986358531e-05,
+      "loss": 1.305,
+      "theoretical_loss": 3.431710974587488,
+      "tokens_seen": 2201944064
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.721232546942706e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.4316944022383424,
+      "tokens_seen": 2202075136
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.720430107526882e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4316778311517666,
+      "tokens_seen": 2202206208
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.719627668111058e-05,
+      "loss": 1.2101,
+      "theoretical_loss": 3.431661261327589,
+      "tokens_seen": 2202337280
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.718825228695234e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.431644692765638,
+      "tokens_seen": 2202468352
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.71802278927941e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.431628125465743,
+      "tokens_seen": 2202599424
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.717220349863586e-05,
+      "loss": 1.3031,
+      "theoretical_loss": 3.4316115594277323,
+      "tokens_seen": 2202730496
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.716417910447762e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4315949946514346,
+      "tokens_seen": 2202861568
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.715615471031938e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.4315784311366793,
+      "tokens_seen": 2202992640
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.714813031616113e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.431561868883295,
+      "tokens_seen": 2203123712
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.71401059220029e-05,
+      "loss": 1.3378,
+      "theoretical_loss": 3.4315453078911102,
+      "tokens_seen": 2203254784
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.713208152784465e-05,
+      "loss": 1.1825,
+      "theoretical_loss": 3.431528748159954,
+      "tokens_seen": 2203385856
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.712405713368642e-05,
+      "loss": 1.2646,
+      "theoretical_loss": 3.4315121896896557,
+      "tokens_seen": 2203516928
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.711603273952817e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.4314956324800443,
+      "tokens_seen": 2203648000
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.710800834536994e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.4314790765309486,
+      "tokens_seen": 2203779072
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.709998395121169e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4314625218421977,
+      "tokens_seen": 2203910144
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.709195955705344e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.431445968413621,
+      "tokens_seen": 2204041216
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.708393516289521e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.431429416245047,
+      "tokens_seen": 2204172288
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.707591076873696e-05,
+      "loss": 1.3295,
+      "theoretical_loss": 3.4314128653363056,
+      "tokens_seen": 2204303360
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.706788637457873e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.4313963156872256,
+      "tokens_seen": 2204434432
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.705986198042048e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.4313797672976363,
+      "tokens_seen": 2204565504
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.705183758626225e-05,
+      "loss": 1.3458,
+      "theoretical_loss": 3.4313632201673667,
+      "tokens_seen": 2204696576
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.7043813192104e-05,
+      "loss": 1.2651,
+      "theoretical_loss": 3.431346674296247,
+      "tokens_seen": 2204827648
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.703578879794575e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.4313301296841052,
+      "tokens_seen": 2204958720
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.702776440378752e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4313135863307718,
+      "tokens_seen": 2205089792
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.701974000962927e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.431297044236076,
+      "tokens_seen": 2205220864
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.701171561547104e-05,
+      "loss": 1.2755,
+      "theoretical_loss": 3.431280503399847,
+      "tokens_seen": 2205351936
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.70036912213128e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.4312639638219142,
+      "tokens_seen": 2205483008
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.699566682715456e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4312474255021073,
+      "tokens_seen": 2205614080
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.698764243299631e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.431230888440256,
+      "tokens_seen": 2205745152
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.697961803883807e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.4312143526361893,
+      "tokens_seen": 2205876224
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.697159364467983e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.4311978180897373,
+      "tokens_seen": 2206007296
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.696356925052159e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4311812848007297,
+      "tokens_seen": 2206138368
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.695554485636335e-05,
+      "loss": 1.313,
+      "theoretical_loss": 3.431164752768996,
+      "tokens_seen": 2206269440
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.694752046220511e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.4311482219943654,
+      "tokens_seen": 2206400512
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.693949606804687e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.4311316924766686,
+      "tokens_seen": 2206531584
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.693147167388863e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.431115164215735,
+      "tokens_seen": 2206662656
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.69234472797304e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.431098637211394,
+      "tokens_seen": 2206793728
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.691542288557215e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.4310821114634757,
+      "tokens_seen": 2206924800
+    },
+    {
+      "epoch": 0.34,
+      "objective/train/docs_used": 742158,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.281986951828003,
+      "objective/train/theoretical_loss": 3.43106558697181,
+      "objective/train/tokens_used": 577514976,
+      "theoretical_loss": 3.43106558697181,
+      "tokens_seen": 2207055872
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.69073984914139e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.43106558697181,
+      "tokens_seen": 2207055872
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.689937409725567e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.4310490637362268,
+      "tokens_seen": 2207186944
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.689134970309742e-05,
+      "loss": 1.2679,
+      "theoretical_loss": 3.431032541756556,
+      "tokens_seen": 2207318016
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.688332530893919e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.4310160210326277,
+      "tokens_seen": 2207449088
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.687530091478094e-05,
+      "loss": 1.29,
+      "theoretical_loss": 3.4309995015642722,
+      "tokens_seen": 2207580160
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.68672765206227e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.430982983351319,
+      "tokens_seen": 2207711232
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.685925212646446e-05,
+      "loss": 1.3048,
+      "theoretical_loss": 3.430966466393598,
+      "tokens_seen": 2207842304
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.685122773230621e-05,
+      "loss": 1.2813,
+      "theoretical_loss": 3.4309499506909398,
+      "tokens_seen": 2207973376
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.684320333814798e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.4309334362431745,
+      "tokens_seen": 2208104448
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.683517894398973e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.430916923050132,
+      "tokens_seen": 2208235520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.68271545498315e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.430900411111643,
+      "tokens_seen": 2208366592
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.681913015567325e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.430883900427537,
+      "tokens_seen": 2208497664
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.681110576151502e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.430867390997645,
+      "tokens_seen": 2208628736
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.680308136735677e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.4308508828217965,
+      "tokens_seen": 2208759808
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.679505697319852e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4308343758998223,
+      "tokens_seen": 2208890880
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.678703257904029e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4308178702315533,
+      "tokens_seen": 2209021952
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.677900818488204e-05,
+      "loss": 1.3539,
+      "theoretical_loss": 3.430801365816819,
+      "tokens_seen": 2209153024
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.677098379072381e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.4307848626554502,
+      "tokens_seen": 2209284096
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.676295939656556e-05,
+      "loss": 1.3167,
+      "theoretical_loss": 3.4307683607472774,
+      "tokens_seen": 2209415168
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.675493500240733e-05,
+      "loss": 1.3084,
+      "theoretical_loss": 3.4307518600921307,
+      "tokens_seen": 2209546240
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.674691060824908e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.430735360689842,
+      "tokens_seen": 2209677312
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.673888621409084e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.43071886254024,
+      "tokens_seen": 2209808384
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.67308618199326e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.4307023656431563,
+      "tokens_seen": 2209939456
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.672283742577436e-05,
+      "loss": 1.3304,
+      "theoretical_loss": 3.4306858699984213,
+      "tokens_seen": 2210070528
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.671481303161612e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.430669375605866,
+      "tokens_seen": 2210201600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.670678863745788e-05,
+      "loss": 1.1526,
+      "theoretical_loss": 3.4306528824653206,
+      "tokens_seen": 2210332672
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.669876424329964e-05,
+      "loss": 1.1759,
+      "theoretical_loss": 3.430636390576616,
+      "tokens_seen": 2210463744
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.66907398491414e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.4306198999395834,
+      "tokens_seen": 2210594816
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.668271545498315e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.4306034105540535,
+      "tokens_seen": 2210725888
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.667469106082492e-05,
+      "loss": 1.2438,
+      "theoretical_loss": 3.4305869224198564,
+      "tokens_seen": 2210856960
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.3122,
+      "theoretical_loss": 3.4305704355368234,
+      "tokens_seen": 2210988032
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.665864227250844e-05,
+      "loss": 1.1834,
+      "theoretical_loss": 3.4305539499047857,
+      "tokens_seen": 2211119104
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.665061787835019e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.4305374655235736,
+      "tokens_seen": 2211250176
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.664259348419196e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.430520982393019,
+      "tokens_seen": 2211381248
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.663456909003371e-05,
+      "loss": 1.1569,
+      "theoretical_loss": 3.430504500512952,
+      "tokens_seen": 2211512320
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.662654469587546e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.430488019883204,
+      "tokens_seen": 2211643392
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.661852030171723e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4304715405036057,
+      "tokens_seen": 2211774464
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.661049590755898e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4304550623739885,
+      "tokens_seen": 2211905536
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.660247151340075e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.4304385854941835,
+      "tokens_seen": 2212036608
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.65944471192425e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.430422109864022,
+      "tokens_seen": 2212167680
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.658642272508427e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.4304056354833357,
+      "tokens_seen": 2212298752
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.657839833092602e-05,
+      "loss": 1.2004,
+      "theoretical_loss": 3.430389162351954,
+      "tokens_seen": 2212429824
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.657037393676779e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.43037269046971,
+      "tokens_seen": 2212560896
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.656234954260954e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.4303562198364346,
+      "tokens_seen": 2212691968
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.65543251484513e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.4303397504519584,
+      "tokens_seen": 2212823040
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.654630075429306e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.4303232823161136,
+      "tokens_seen": 2212954112
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.653827636013481e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.4303068154287306,
+      "tokens_seen": 2213085184
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.653025196597658e-05,
+      "loss": 1.3145,
+      "theoretical_loss": 3.430290349789642,
+      "tokens_seen": 2213216256
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.652222757181833e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.4302738853986785,
+      "tokens_seen": 2213347328
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.65142031776601e-05,
+      "loss": 1.2938,
+      "theoretical_loss": 3.430257422255672,
+      "tokens_seen": 2213478400
+    },
+    {
+      "epoch": 0.34,
+      "objective/train/docs_used": 744341,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1990547180175781,
+      "objective/train/theoretical_loss": 3.4302409603604533,
+      "objective/train/tokens_used": 584068576,
+      "theoretical_loss": 3.4302409603604533,
+      "tokens_seen": 2213609472
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.650617878350185e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.4302409603604533,
+      "tokens_seen": 2213609472
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.649815438934361e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.4302244997128546,
+      "tokens_seen": 2213740544
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.649012999518537e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.4302080403127073,
+      "tokens_seen": 2213871616
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.648210560102713e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.4301915821598428,
+      "tokens_seen": 2214002688
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.64740812068689e-05,
+      "loss": 1.2184,
+      "theoretical_loss": 3.4301751252540935,
+      "tokens_seen": 2214133760
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.646605681271065e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.43015866959529,
+      "tokens_seen": 2214264832
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.645803241855241e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.430142215183265,
+      "tokens_seen": 2214395904
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.645000802439417e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.4301257620178496,
+      "tokens_seen": 2214526976
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.644198363023592e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.4301093100988758,
+      "tokens_seen": 2214658048
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.643395923607769e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.4300928594261757,
+      "tokens_seen": 2214789120
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.642593484191944e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4300764099995806,
+      "tokens_seen": 2214920192
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.64179104477612e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4300599618189223,
+      "tokens_seen": 2215051264
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.640988605360296e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.4300435148840336,
+      "tokens_seen": 2215182336
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.640186165944471e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4300270691947454,
+      "tokens_seen": 2215313408
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.639383726528648e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.430010624750891,
+      "tokens_seen": 2215444480
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.638581287112823e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.429994181552301,
+      "tokens_seen": 2215575552
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.637778847697e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.429977739598808,
+      "tokens_seen": 2215706624
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.636976408281175e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4299612988902437,
+      "tokens_seen": 2215837696
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.636173968865352e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.429944859426441,
+      "tokens_seen": 2215968768
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.635371529449527e-05,
+      "loss": 1.1652,
+      "theoretical_loss": 3.4299284212072316,
+      "tokens_seen": 2216099840
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.634569090033703e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.429911984232448,
+      "tokens_seen": 2216230912
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.633766650617879e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.429895548501922,
+      "tokens_seen": 2216361984
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.632964211202055e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.4298791140154856,
+      "tokens_seen": 2216493056
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.632161771786231e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4298626807729713,
+      "tokens_seen": 2216624128
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.631359332370407e-05,
+      "loss": 1.1913,
+      "theoretical_loss": 3.4298462487742123,
+      "tokens_seen": 2216755200
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.630556892954582e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.4298298180190394,
+      "tokens_seen": 2216886272
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.629754453538758e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.429813388507286,
+      "tokens_seen": 2217017344
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.628952014122934e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.429796960238784,
+      "tokens_seen": 2217148416
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.62814957470711e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.4297805332133664,
+      "tokens_seen": 2217279488
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.627347135291286e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.429764107430865,
+      "tokens_seen": 2217410560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.626544695875462e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.429747682891113,
+      "tokens_seen": 2217541632
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.625742256459638e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.429731259593942,
+      "tokens_seen": 2217672704
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.624939817043813e-05,
+      "loss": 1.3025,
+      "theoretical_loss": 3.4297148375391853,
+      "tokens_seen": 2217803776
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.62413737762799e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.4296984167266755,
+      "tokens_seen": 2217934848
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.623334938212165e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.4296819971562447,
+      "tokens_seen": 2218065920
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.622532498796342e-05,
+      "loss": 1.3204,
+      "theoretical_loss": 3.429665578827726,
+      "tokens_seen": 2218196992
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.621730059380517e-05,
+      "loss": 1.1782,
+      "theoretical_loss": 3.429649161740952,
+      "tokens_seen": 2218328064
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.620927619964692e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.4296327458957556,
+      "tokens_seen": 2218459136
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.620125180548869e-05,
+      "loss": 1.2784,
+      "theoretical_loss": 3.429616331291969,
+      "tokens_seen": 2218590208
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.619322741133044e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4295999179294254,
+      "tokens_seen": 2218721280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.618520301717221e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.429583505807958,
+      "tokens_seen": 2218852352
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.617717862301396e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4295670949273984,
+      "tokens_seen": 2218983424
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.616915422885573e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.429550685287581,
+      "tokens_seen": 2219114496
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.616112983469748e-05,
+      "loss": 1.2902,
+      "theoretical_loss": 3.4295342768883375,
+      "tokens_seen": 2219245568
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.615310544053924e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.4295178697295015,
+      "tokens_seen": 2219376640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.6145081046381e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.429501463810906,
+      "tokens_seen": 2219507712
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.613705665222276e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4294850591323836,
+      "tokens_seen": 2219638784
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.612903225806452e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.4294686556937677,
+      "tokens_seen": 2219769856
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.612100786390628e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4294522534948912,
+      "tokens_seen": 2219900928
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.611298346974803e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.4294358525355877,
+      "tokens_seen": 2220032000
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 746497,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.260611653327942,
+      "objective/train/theoretical_loss": 3.4294194528156896,
+      "objective/train/tokens_used": 590622176,
+      "theoretical_loss": 3.4294194528156896,
+      "tokens_seen": 2220163072
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.61049590755898e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.4294194528156896,
+      "tokens_seen": 2220163072
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.609693468143155e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.4294030543350305,
+      "tokens_seen": 2220294144
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.608891028727332e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.4293866570934437,
+      "tokens_seen": 2220425216
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.608088589311507e-05,
+      "loss": 1.3183,
+      "theoretical_loss": 3.429370261090762,
+      "tokens_seen": 2220556288
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.607286149895684e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.429353866326819,
+      "tokens_seen": 2220687360
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.606483710479859e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.429337472801448,
+      "tokens_seen": 2220818432
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.605681271064034e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.429321080514482,
+      "tokens_seen": 2220949504
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.604878831648211e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.429304689465755,
+      "tokens_seen": 2221080576
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.604076392232386e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.4292882996551,
+      "tokens_seen": 2221211648
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.603273952816563e-05,
+      "loss": 1.1655,
+      "theoretical_loss": 3.4292719110823504,
+      "tokens_seen": 2221342720
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.602471513400738e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.4292555237473397,
+      "tokens_seen": 2221473792
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.601669073984913e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.4292391376499016,
+      "tokens_seen": 2221604864
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.60086663456909e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.4292227527898693,
+      "tokens_seen": 2221735936
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.600064195153265e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.429206369167077,
+      "tokens_seen": 2221867008
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.599261755737442e-05,
+      "loss": 1.3114,
+      "theoretical_loss": 3.429189986781357,
+      "tokens_seen": 2221998080
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.598459316321617e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4291736056325446,
+      "tokens_seen": 2222129152
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.597656876905794e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.4291572257204725,
+      "tokens_seen": 2222260224
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.59685443748997e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4291408470449745,
+      "tokens_seen": 2222391296
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.596051998074145e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.429124469605884,
+      "tokens_seen": 2222522368
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.595249558658321e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.4291080934030354,
+      "tokens_seen": 2222653440
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.594447119242497e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.429091718436262,
+      "tokens_seen": 2222784512
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.593644679826673e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.429075344705398,
+      "tokens_seen": 2222915584
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.592842240410849e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.429058972210277,
+      "tokens_seen": 2223046656
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.592039800995025e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.429042600950733,
+      "tokens_seen": 2223177728
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.5912373615792e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4290262309266,
+      "tokens_seen": 2223308800
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.590434922163376e-05,
+      "loss": 1.3366,
+      "theoretical_loss": 3.4290098621377116,
+      "tokens_seen": 2223439872
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.589632482747553e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.428993494583902,
+      "tokens_seen": 2223570944
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.588830043331728e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.4289771282650054,
+      "tokens_seen": 2223702016
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.588027603915905e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.4289607631808554,
+      "tokens_seen": 2223833088
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.58722516450008e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.4289443993312867,
+      "tokens_seen": 2223964160
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.586422725084257e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4289280367161328,
+      "tokens_seen": 2224095232
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.585620285668432e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.428911675335228,
+      "tokens_seen": 2224226304
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.584817846252607e-05,
+      "loss": 1.3029,
+      "theoretical_loss": 3.428895315188407,
+      "tokens_seen": 2224357376
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.584015406836784e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.4288789562755033,
+      "tokens_seen": 2224488448
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.583212967420959e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.4288625985963517,
+      "tokens_seen": 2224619520
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.582410528005136e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4288462421507857,
+      "tokens_seen": 2224750592
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.581608088589311e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.42882988693864,
+      "tokens_seen": 2224881664
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.580805649173488e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.4288135329597496,
+      "tokens_seen": 2225012736
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.580003209757663e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.428797180213948,
+      "tokens_seen": 2225143808
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.579200770341838e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.42878082870107,
+      "tokens_seen": 2225274880
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.578398330926015e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.4287644784209497,
+      "tokens_seen": 2225405952
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.57759589151019e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.428748129373422,
+      "tokens_seen": 2225537024
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.576793452094367e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.428731781558321,
+      "tokens_seen": 2225668096
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.575991012678542e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.428715434975482,
+      "tokens_seen": 2225799168
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.575188573262719e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.428699089624738,
+      "tokens_seen": 2225930240
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.574386133846894e-05,
+      "loss": 1.3067,
+      "theoretical_loss": 3.428682745505925,
+      "tokens_seen": 2226061312
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.57358369443107e-05,
+      "loss": 1.2855,
+      "theoretical_loss": 3.428666402618877,
+      "tokens_seen": 2226192384
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.572781255015246e-05,
+      "loss": 1.2746,
+      "theoretical_loss": 3.428650060963429,
+      "tokens_seen": 2226323456
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.571978815599422e-05,
+      "loss": 1.2653,
+      "theoretical_loss": 3.4286337205394153,
+      "tokens_seen": 2226454528
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.571176376183598e-05,
+      "loss": 1.2856,
+      "theoretical_loss": 3.428617381346671,
+      "tokens_seen": 2226585600
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 748781,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.189249873161316,
+      "objective/train/theoretical_loss": 3.428601043385031,
+      "objective/train/tokens_used": 597175776,
+      "theoretical_loss": 3.428601043385031,
+      "tokens_seen": 2226716672
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.570373936767774e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.428601043385031,
+      "tokens_seen": 2226716672
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.56957149735195e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.4285847066543296,
+      "tokens_seen": 2226847744
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.568769057936126e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.4285683711544013,
+      "tokens_seen": 2226978816
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.567966618520301e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.428552036885082,
+      "tokens_seen": 2227109888
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.567164179104478e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.4285357038462054,
+      "tokens_seen": 2227240960
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.566361739688653e-05,
+      "loss": 1.3279,
+      "theoretical_loss": 3.4285193720376075,
+      "tokens_seen": 2227372032
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.56555930027283e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.4285030414591224,
+      "tokens_seen": 2227503104
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.564756860857005e-05,
+      "loss": 1.2847,
+      "theoretical_loss": 3.428486712110586,
+      "tokens_seen": 2227634176
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.563954421441182e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.428470383991832,
+      "tokens_seen": 2227765248
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.563151982025357e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.4284540571026967,
+      "tokens_seen": 2227896320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.562349542609534e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.4284377314430143,
+      "tokens_seen": 2228027392
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.561547103193709e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.42842140701262,
+      "tokens_seen": 2228158464
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.560744663777884e-05,
+      "loss": 1.3266,
+      "theoretical_loss": 3.42840508381135,
+      "tokens_seen": 2228289536
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.559942224362061e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.428388761839038,
+      "tokens_seen": 2228420608
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.559139784946236e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.42837244109552,
+      "tokens_seen": 2228551680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.558337345530413e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.428356121580631,
+      "tokens_seen": 2228682752
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.557534906114588e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.4283398032942065,
+      "tokens_seen": 2228813824
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.556732466698765e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.4283234862360814,
+      "tokens_seen": 2228944896
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.55593002728294e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.4283071704060912,
+      "tokens_seen": 2229075968
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.555127587867115e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4282908558040712,
+      "tokens_seen": 2229207040
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.554325148451292e-05,
+      "loss": 1.2948,
+      "theoretical_loss": 3.4282745424298566,
+      "tokens_seen": 2229338112
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.553522709035467e-05,
+      "loss": 1.2734,
+      "theoretical_loss": 3.4282582302832836,
+      "tokens_seen": 2229469184
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.552720269619644e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.4282419193641873,
+      "tokens_seen": 2229600256
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.55191783020382e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.4282256096724026,
+      "tokens_seen": 2229731328
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.551115390787996e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.4282093012077652,
+      "tokens_seen": 2229862400
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.550312951372171e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.4281929939701112,
+      "tokens_seen": 2229993472
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.549510511956347e-05,
+      "loss": 1.2645,
+      "theoretical_loss": 3.428176687959276,
+      "tokens_seen": 2230124544
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.548708072540523e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.428160383175095,
+      "tokens_seen": 2230255616
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.547905633124699e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.428144079617404,
+      "tokens_seen": 2230386688
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.547103193708875e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.428127777286038,
+      "tokens_seen": 2230517760
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.54630075429305e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.428111476180834,
+      "tokens_seen": 2230648832
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.545498314877227e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.428095176301626,
+      "tokens_seen": 2230779904
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.544695875461403e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.4280788776482516,
+      "tokens_seen": 2230910976
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.543893436045578e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.4280625802205456,
+      "tokens_seen": 2231042048
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.543090996629755e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.428046284018344,
+      "tokens_seen": 2231173120
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.54228855721393e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.4280299890414825,
+      "tokens_seen": 2231304192
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.541486117798107e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.4280136952897973,
+      "tokens_seen": 2231435264
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.540683678382282e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.427997402763124,
+      "tokens_seen": 2231566336
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.539881238966459e-05,
+      "loss": 1.2586,
+      "theoretical_loss": 3.427981111461299,
+      "tokens_seen": 2231697408
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.539078799550634e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.4279648213841583,
+      "tokens_seen": 2231828480
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.538276360134809e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.427948532531537,
+      "tokens_seen": 2231959552
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.537473920718986e-05,
+      "loss": 1.3261,
+      "theoretical_loss": 3.427932244903272,
+      "tokens_seen": 2232090624
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.536671481303161e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.427915958499199,
+      "tokens_seen": 2232221696
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.535869041887338e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.4278996733191547,
+      "tokens_seen": 2232352768
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.535066602471513e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4278833893629748,
+      "tokens_seen": 2232483840
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.53426416305569e-05,
+      "loss": 1.3497,
+      "theoretical_loss": 3.4278671066304955,
+      "tokens_seen": 2232614912
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.533461723639865e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.4278508251215527,
+      "tokens_seen": 2232745984
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.532659284224042e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4278345448359833,
+      "tokens_seen": 2232877056
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.531856844808217e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.427818265773623,
+      "tokens_seen": 2233008128
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.531054405392392e-05,
+      "loss": 1.312,
+      "theoretical_loss": 3.4278019879343087,
+      "tokens_seen": 2233139200
+    },
+    {
+      "epoch": 0.35,
+      "objective/train/docs_used": 751008,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2541487216949463,
+      "objective/train/theoretical_loss": 3.427785711317876,
+      "objective/train/tokens_used": 603729376,
+      "theoretical_loss": 3.427785711317876,
+      "tokens_seen": 2233270272
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.530251965976569e-05,
+      "loss": 1.2628,
+      "theoretical_loss": 3.427785711317876,
+      "tokens_seen": 2233270272
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.529449526560744e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4277694359241617,
+      "tokens_seen": 2233401344
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.528647087144921e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.427753161753002,
+      "tokens_seen": 2233532416
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.527844647729096e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.427736888804234,
+      "tokens_seen": 2233663488
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.527042208313273e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.4277206170776933,
+      "tokens_seen": 2233794560
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.526239768897448e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4277043465732167,
+      "tokens_seen": 2233925632
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.525437329481624e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.427688077290641,
+      "tokens_seen": 2234056704
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.5246348900658e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.4276718092298024,
+      "tokens_seen": 2234187776
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.523832450649976e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.4276555423905375,
+      "tokens_seen": 2234318848
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.523030011234152e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.4276392767726827,
+      "tokens_seen": 2234449920
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.522227571818328e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.4276230123760754,
+      "tokens_seen": 2234580992
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.521425132402504e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.4276067492005518,
+      "tokens_seen": 2234712064
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.52062269298668e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.4275904872459484,
+      "tokens_seen": 2234843136
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.519820253570855e-05,
+      "loss": 1.3077,
+      "theoretical_loss": 3.4275742265121023,
+      "tokens_seen": 2234974208
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.519017814155032e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.42755796699885,
+      "tokens_seen": 2235105280
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.518215374739207e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.427541708706029,
+      "tokens_seen": 2235236352
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.517412935323384e-05,
+      "loss": 1.2902,
+      "theoretical_loss": 3.427525451633475,
+      "tokens_seen": 2235367424
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.516610495907559e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.427509195781026,
+      "tokens_seen": 2235498496
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.515808056491736e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.427492941148518,
+      "tokens_seen": 2235629568
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.515005617075911e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.4274766877357887,
+      "tokens_seen": 2235760640
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.514203177660086e-05,
+      "loss": 1.289,
+      "theoretical_loss": 3.4274604355426748,
+      "tokens_seen": 2235891712
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.513400738244263e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.427444184569013,
+      "tokens_seen": 2236022784
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.512598298828438e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.42742793481464,
+      "tokens_seen": 2236153856
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.511795859412615e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.427411686279394,
+      "tokens_seen": 2236284928
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.51099341999679e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.4273954389631114,
+      "tokens_seen": 2236416000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.510190980580967e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.4273791928656294,
+      "tokens_seen": 2236547072
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.509388541165142e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.4273629479867846,
+      "tokens_seen": 2236678144
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.508586101749317e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.427346704326415,
+      "tokens_seen": 2236809216
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.507783662333494e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.427330461884358,
+      "tokens_seen": 2236940288
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.50698122291767e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.42731422066045,
+      "tokens_seen": 2237071360
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.506178783501846e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.4272979806545285,
+      "tokens_seen": 2237202432
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.505376344086021e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4272817418664308,
+      "tokens_seen": 2237333504
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.504573904670198e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.4272655042959945,
+      "tokens_seen": 2237464576
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.503771465254373e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.4272492679430573,
+      "tokens_seen": 2237595648
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.50296902583855e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4272330328074556,
+      "tokens_seen": 2237726720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.502166586422725e-05,
+      "loss": 1.3199,
+      "theoretical_loss": 3.427216798889028,
+      "tokens_seen": 2237857792
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.501364147006901e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.427200566187611,
+      "tokens_seen": 2237988864
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.500561707591077e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.4271843347030426,
+      "tokens_seen": 2238119936
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.499759268175253e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.42716810443516,
+      "tokens_seen": 2238251008
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.49895682875943e-05,
+      "loss": 1.2987,
+      "theoretical_loss": 3.4271518753838013,
+      "tokens_seen": 2238382080
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.498154389343605e-05,
+      "loss": 1.317,
+      "theoretical_loss": 3.4271356475488033,
+      "tokens_seen": 2238513152
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.497351949927781e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.427119420930004,
+      "tokens_seen": 2238644224
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.496549510511957e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4271031955272413,
+      "tokens_seen": 2238775296
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.495747071096132e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.427086971340353,
+      "tokens_seen": 2238906368
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.494944631680309e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.427070748369176,
+      "tokens_seen": 2239037440
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.494142192264484e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.427054526613549,
+      "tokens_seen": 2239168512
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.49333975284866e-05,
+      "loss": 1.1808,
+      "theoretical_loss": 3.427038306073309,
+      "tokens_seen": 2239299584
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.492537313432836e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.4270220867482943,
+      "tokens_seen": 2239430656
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.491734874017013e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4270058686383424,
+      "tokens_seen": 2239561728
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.490932434601188e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.4269896517432916,
+      "tokens_seen": 2239692800
+    },
+    {
+      "epoch": 0.36,
+      "objective/train/docs_used": 752999,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1579296588897705,
+      "objective/train/theoretical_loss": 3.4269734360629793,
+      "objective/train/tokens_used": 610282976,
+      "theoretical_loss": 3.4269734360629793,
+      "tokens_seen": 2239823872
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.490129995185363e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.4269734360629793,
+      "tokens_seen": 2239823872
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.48932755576954e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.4269572215972435,
+      "tokens_seen": 2239954944
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.488525116353715e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.426941008345923,
+      "tokens_seen": 2240086016
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.487722676937892e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4269247963088545,
+      "tokens_seen": 2240217088
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.486920237522067e-05,
+      "loss": 1.3471,
+      "theoretical_loss": 3.4269085854858767,
+      "tokens_seen": 2240348160
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.486117798106244e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.426892375876828,
+      "tokens_seen": 2240479232
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.485315358690419e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.426876167481546,
+      "tokens_seen": 2240610304
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.484512919274594e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.426859960299869,
+      "tokens_seen": 2240741376
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.483710479858771e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4268437543316344,
+      "tokens_seen": 2240872448
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.482908040442946e-05,
+      "loss": 1.3203,
+      "theoretical_loss": 3.426827549576682,
+      "tokens_seen": 2241003520
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.482105601027123e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.426811346034848,
+      "tokens_seen": 2241134592
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.481303161611298e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.4267951437059727,
+      "tokens_seen": 2241265664
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.480500722195475e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.426778942589893,
+      "tokens_seen": 2241396736
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.47969828277965e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.4267627426864475,
+      "tokens_seen": 2241527808
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.478895843363826e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.4267465439954745,
+      "tokens_seen": 2241658880
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.478093403948002e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.4267303465168126,
+      "tokens_seen": 2241789952
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.477290964532178e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4267141502503002,
+      "tokens_seen": 2241921024
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.476488525116354e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.4266979551957757,
+      "tokens_seen": 2242052096
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.47568608570053e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.426681761353077,
+      "tokens_seen": 2242183168
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.474883646284706e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4266655687220435,
+      "tokens_seen": 2242314240
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.474081206868882e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.426649377302513,
+      "tokens_seen": 2242445312
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.473278767453057e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.4266331870943247,
+      "tokens_seen": 2242576384
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.472476328037234e-05,
+      "loss": 1.3374,
+      "theoretical_loss": 3.4266169980973165,
+      "tokens_seen": 2242707456
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.471673888621409e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.4266008103113275,
+      "tokens_seen": 2242838528
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.470871449205586e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.426584623736196,
+      "tokens_seen": 2242969600
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.470069009789761e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4265684383717607,
+      "tokens_seen": 2243100672
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.469266570373938e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.426552254217861,
+      "tokens_seen": 2243231744
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.468464130958113e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.4265360712743345,
+      "tokens_seen": 2243362816
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.46766169154229e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.426519889541021,
+      "tokens_seen": 2243493888
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.466859252126465e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.4265037090177586,
+      "tokens_seen": 2243624960
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.46605681271064e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.426487529704387,
+      "tokens_seen": 2243756032
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.465254373294817e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.426471351600744,
+      "tokens_seen": 2243887104
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.464451933878992e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.4264551747066685,
+      "tokens_seen": 2244018176
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.463649494463169e-05,
+      "loss": 1.1834,
+      "theoretical_loss": 3.4264389990220003,
+      "tokens_seen": 2244149248
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.462847055047344e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.426422824546578,
+      "tokens_seen": 2244280320
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.462044615631521e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4264066512802405,
+      "tokens_seen": 2244411392
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.461242176215696e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.426390479222827,
+      "tokens_seen": 2244542464
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.460439736799871e-05,
+      "loss": 1.3113,
+      "theoretical_loss": 3.426374308374176,
+      "tokens_seen": 2244673536
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.459637297384048e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4263581387341273,
+      "tokens_seen": 2244804608
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.458834857968223e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.426341970302519,
+      "tokens_seen": 2244935680
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.4580324185524e-05,
+      "loss": 1.1771,
+      "theoretical_loss": 3.426325803079192,
+      "tokens_seen": 2245066752
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.457229979136575e-05,
+      "loss": 1.2893,
+      "theoretical_loss": 3.4263096370639836,
+      "tokens_seen": 2245197824
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.456427539720752e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.426293472256734,
+      "tokens_seen": 2245328896
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.455625100304927e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.4262773086572818,
+      "tokens_seen": 2245459968
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.454822660889103e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.426261146265467,
+      "tokens_seen": 2245591040
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.45402022147328e-05,
+      "loss": 1.3281,
+      "theoretical_loss": 3.4262449850811287,
+      "tokens_seen": 2245722112
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.453217782057455e-05,
+      "loss": 1.3219,
+      "theoretical_loss": 3.426228825104106,
+      "tokens_seen": 2245853184
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.452415342641631e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.4262126663342385,
+      "tokens_seen": 2245984256
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.451612903225807e-05,
+      "loss": 1.309,
+      "theoretical_loss": 3.426196508771365,
+      "tokens_seen": 2246115328
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.450810463809983e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.426180352415326,
+      "tokens_seen": 2246246400
+    },
+    {
+      "epoch": 0.36,
+      "objective/train/docs_used": 755049,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2689517736434937,
+      "objective/train/theoretical_loss": 3.4261641972659596,
+      "objective/train/tokens_used": 616836576,
+      "theoretical_loss": 3.4261641972659596,
+      "tokens_seen": 2246377472
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.450008024394159e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.4261641972659596,
+      "tokens_seen": 2246377472
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.449205584978334e-05,
+      "loss": 1.2136,
+      "theoretical_loss": 3.426148043323106,
+      "tokens_seen": 2246508544
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.448403145562511e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.426131890586605,
+      "tokens_seen": 2246639616
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.447600706146686e-05,
+      "loss": 1.307,
+      "theoretical_loss": 3.4261157390562955,
+      "tokens_seen": 2246770688
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.446798266730863e-05,
+      "loss": 1.3096,
+      "theoretical_loss": 3.4260995887320176,
+      "tokens_seen": 2246901760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.445995827315038e-05,
+      "loss": 1.298,
+      "theoretical_loss": 3.4260834396136106,
+      "tokens_seen": 2247032832
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.445193387899215e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.4260672917009147,
+      "tokens_seen": 2247163904
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.44439094848339e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.4260511449937683,
+      "tokens_seen": 2247294976
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.443588509067565e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4260349994920123,
+      "tokens_seen": 2247426048
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.442786069651742e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.426018855195486,
+      "tokens_seen": 2247557120
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.441983630235917e-05,
+      "loss": 1.2802,
+      "theoretical_loss": 3.426002712104029,
+      "tokens_seen": 2247688192
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.441181190820094e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.425986570217481,
+      "tokens_seen": 2247819264
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.440378751404269e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.425970429535683,
+      "tokens_seen": 2247950336
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.439576311988446e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.425954290058473,
+      "tokens_seen": 2248081408
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.438773872572621e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.425938151785692,
+      "tokens_seen": 2248212480
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.437971433156798e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.4259220147171794,
+      "tokens_seen": 2248343552
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.437168993740973e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4259058788527756,
+      "tokens_seen": 2248474624
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.436366554325149e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.425889744192321,
+      "tokens_seen": 2248605696
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.435564114909325e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.425873610735654,
+      "tokens_seen": 2248736768
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.4347616754935e-05,
+      "loss": 1.3048,
+      "theoretical_loss": 3.4258574784826163,
+      "tokens_seen": 2248867840
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.433959236077677e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.425841347433047,
+      "tokens_seen": 2248998912
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.433156796661852e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.4258252175867865,
+      "tokens_seen": 2249129984
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.432354357246029e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.4258090889436748,
+      "tokens_seen": 2249261056
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.431551917830204e-05,
+      "loss": 1.2036,
+      "theoretical_loss": 3.4257929615035523,
+      "tokens_seen": 2249392128
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.43074947841438e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4257768352662588,
+      "tokens_seen": 2249523200
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.429947038998556e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4257607102316348,
+      "tokens_seen": 2249654272
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.429144599582732e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.425744586399521,
+      "tokens_seen": 2249785344
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.428342160166908e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.4257284637697563,
+      "tokens_seen": 2249916416
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.427539720751084e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.425712342342182,
+      "tokens_seen": 2250047488
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.42673728133526e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4256962221166383,
+      "tokens_seen": 2250178560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.425934841919436e-05,
+      "loss": 1.3509,
+      "theoretical_loss": 3.425680103092966,
+      "tokens_seen": 2250309632
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.425132402503611e-05,
+      "loss": 1.3128,
+      "theoretical_loss": 3.4256639852710045,
+      "tokens_seen": 2250440704
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.424329963087788e-05,
+      "loss": 1.1881,
+      "theoretical_loss": 3.425647868650595,
+      "tokens_seen": 2250571776
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.423527523671963e-05,
+      "loss": 1.3056,
+      "theoretical_loss": 3.425631753231578,
+      "tokens_seen": 2250702848
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.42272508425614e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.4256156390137935,
+      "tokens_seen": 2250833920
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.421922644840315e-05,
+      "loss": 1.1545,
+      "theoretical_loss": 3.4255995259970824,
+      "tokens_seen": 2250964992
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.421120205424492e-05,
+      "loss": 1.2797,
+      "theoretical_loss": 3.425583414181285,
+      "tokens_seen": 2251096064
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.420317766008667e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.4255673035662415,
+      "tokens_seen": 2251227136
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.419515326592842e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.4255511941517933,
+      "tokens_seen": 2251358208
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.418712887177019e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.425535085937781,
+      "tokens_seen": 2251489280
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.417910447761194e-05,
+      "loss": 1.3065,
+      "theoretical_loss": 3.4255189789240443,
+      "tokens_seen": 2251620352
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.417108008345371e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.425502873110425,
+      "tokens_seen": 2251751424
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.416305568929546e-05,
+      "loss": 1.1835,
+      "theoretical_loss": 3.4254867684967634,
+      "tokens_seen": 2251882496
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.415503129513723e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.4254706650829005,
+      "tokens_seen": 2252013568
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 6.414700690097898e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4254545628686768,
+      "tokens_seen": 2252144640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.413898250682074e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.4254384618539326,
+      "tokens_seen": 2252275712
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.41309581126625e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.42542236203851,
+      "tokens_seen": 2252406784
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.412293371850426e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.425406263422249,
+      "tokens_seen": 2252537856
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.411490932434602e-05,
+      "loss": 1.1744,
+      "theoretical_loss": 3.425390166004991,
+      "tokens_seen": 2252668928
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.410688493018777e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.425374069786576,
+      "tokens_seen": 2252800000
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 757120,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2600257396697998,
+      "objective/train/theoretical_loss": 3.425357974766846,
+      "objective/train/tokens_used": 623390176,
+      "theoretical_loss": 3.425357974766846,
+      "tokens_seen": 2252931072
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.409886053602954e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.425357974766846,
+      "tokens_seen": 2252931072
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.40908361418713e-05,
+      "loss": 1.3275,
+      "theoretical_loss": 3.4253418809456417,
+      "tokens_seen": 2253062144
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.408281174771305e-05,
+      "loss": 1.3107,
+      "theoretical_loss": 3.4253257883228043,
+      "tokens_seen": 2253193216
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.407478735355481e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.425309696898174,
+      "tokens_seen": 2253324288
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.406676295939657e-05,
+      "loss": 1.334,
+      "theoretical_loss": 3.4252936066715933,
+      "tokens_seen": 2253455360
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.405873856523833e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.4252775176429022,
+      "tokens_seen": 2253586432
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.405071417108009e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4252614298119424,
+      "tokens_seen": 2253717504
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.404268977692185e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.425245343178555,
+      "tokens_seen": 2253848576
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.403466538276361e-05,
+      "loss": 1.1936,
+      "theoretical_loss": 3.4252292577425805,
+      "tokens_seen": 2253979648
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.402664098860537e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4252131735038613,
+      "tokens_seen": 2254110720
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.401861659444713e-05,
+      "loss": 1.295,
+      "theoretical_loss": 3.425197090462238,
+      "tokens_seen": 2254241792
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.401059220028888e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.4251810086175523,
+      "tokens_seen": 2254372864
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.400256780613065e-05,
+      "loss": 1.2837,
+      "theoretical_loss": 3.425164927969645,
+      "tokens_seen": 2254503936
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.39945434119724e-05,
+      "loss": 1.1796,
+      "theoretical_loss": 3.425148848518358,
+      "tokens_seen": 2254635008
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.398651901781417e-05,
+      "loss": 1.1709,
+      "theoretical_loss": 3.425132770263532,
+      "tokens_seen": 2254766080
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.397849462365592e-05,
+      "loss": 1.2768,
+      "theoretical_loss": 3.4251166932050094,
+      "tokens_seen": 2254897152
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.397047022949769e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.4251006173426313,
+      "tokens_seen": 2255028224
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.396244583533944e-05,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4250845426762386,
+      "tokens_seen": 2255159296
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.395442144118119e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.4250684692056734,
+      "tokens_seen": 2255290368
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.394639704702296e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.425052396930777,
+      "tokens_seen": 2255421440
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.393837265286471e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.4250363258513916,
+      "tokens_seen": 2255552512
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.393034825870648e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.4250202559673575,
+      "tokens_seen": 2255683584
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.392232386454823e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.4250041872785175,
+      "tokens_seen": 2255814656
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.391429947039e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.424988119784713,
+      "tokens_seen": 2255945728
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.390627507623175e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.424972053485786,
+      "tokens_seen": 2256076800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.38982506820735e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.424955988381577,
+      "tokens_seen": 2256207872
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.389022628791527e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.4249399244719294,
+      "tokens_seen": 2256338944
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.388220189375703e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.4249238617566835,
+      "tokens_seen": 2256470016
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.387417749959879e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.4249078002356823,
+      "tokens_seen": 2256601088
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.386615310544055e-05,
+      "loss": 1.3636,
+      "theoretical_loss": 3.424891739908767,
+      "tokens_seen": 2256732160
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.385812871128231e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.4248756807757794,
+      "tokens_seen": 2256863232
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.385010431712406e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.424859622836562,
+      "tokens_seen": 2256994304
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.384207992296582e-05,
+      "loss": 1.3186,
+      "theoretical_loss": 3.4248435660909564,
+      "tokens_seen": 2257125376
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.383405552880758e-05,
+      "loss": 1.1643,
+      "theoretical_loss": 3.424827510538804,
+      "tokens_seen": 2257256448
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.382603113464934e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.424811456179948,
+      "tokens_seen": 2257387520
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.38180067404911e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.4247954030142296,
+      "tokens_seen": 2257518592
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.380998234633286e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.424779351041491,
+      "tokens_seen": 2257649664
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.380195795217462e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.4247633002615743,
+      "tokens_seen": 2257780736
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.379393355801638e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.424747250674322,
+      "tokens_seen": 2257911808
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.378590916385813e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.4247312022795757,
+      "tokens_seen": 2258042880
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.37778847696999e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4247151550771777,
+      "tokens_seen": 2258173952
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.376986037554165e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.4246991090669705,
+      "tokens_seen": 2258305024
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.376183598138342e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.424683064248796,
+      "tokens_seen": 2258436096
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.375381158722517e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.424667020622497,
+      "tokens_seen": 2258567168
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.374578719306694e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.4246509781879153,
+      "tokens_seen": 2258698240
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.373776279890869e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.424634936944893,
+      "tokens_seen": 2258829312
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.372973840475044e-05,
+      "loss": 1.3115,
+      "theoretical_loss": 3.424618896893273,
+      "tokens_seen": 2258960384
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.372171401059221e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.4246028580328973,
+      "tokens_seen": 2259091456
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.371368961643396e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.424586820363609,
+      "tokens_seen": 2259222528
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.370566522227573e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.42457078388525,
+      "tokens_seen": 2259353600
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 759180,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.287301778793335,
+      "objective/train/theoretical_loss": 3.4245547485976626,
+      "objective/train/tokens_used": 629943776,
+      "theoretical_loss": 3.4245547485976626,
+      "tokens_seen": 2259484672
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.369764082811748e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.4245547485976626,
+      "tokens_seen": 2259484672
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.368961643395925e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.42453871450069,
+      "tokens_seen": 2259615744
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.3681592039801e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.424522681594174,
+      "tokens_seen": 2259746816
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.367356764564276e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.4245066498779577,
+      "tokens_seen": 2259877888
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.366554325148452e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.424490619351883,
+      "tokens_seen": 2260008960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.365751885732628e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.4244745900157936,
+      "tokens_seen": 2260140032
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.364949446316804e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.424458561869532,
+      "tokens_seen": 2260271104
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.36414700690098e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.4244425349129397,
+      "tokens_seen": 2260402176
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.363344567485155e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.42442650914586,
+      "tokens_seen": 2260533248
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.362542128069332e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.4244104845681367,
+      "tokens_seen": 2260664320
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.361739688653507e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4243944611796113,
+      "tokens_seen": 2260795392
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.360937249237683e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.424378438980127,
+      "tokens_seen": 2260926464
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.360134809821859e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.424362417969527,
+      "tokens_seen": 2261057536
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.359332370406035e-05,
+      "loss": 1.3146,
+      "theoretical_loss": 3.4243463981476534,
+      "tokens_seen": 2261188608
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.358529930990211e-05,
+      "loss": 1.2688,
+      "theoretical_loss": 3.42433037951435,
+      "tokens_seen": 2261319680
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.357727491574386e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4243143620694587,
+      "tokens_seen": 2261450752
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.356925052158563e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.424298345812824,
+      "tokens_seen": 2261581824
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.356122612742738e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.4242823307442873,
+      "tokens_seen": 2261712896
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.355320173326915e-05,
+      "loss": 1.1955,
+      "theoretical_loss": 3.4242663168636924,
+      "tokens_seen": 2261843968
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.35451773391109e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.424250304170882,
+      "tokens_seen": 2261975040
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.353715294495265e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.4242342926657,
+      "tokens_seen": 2262106112
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.352912855079442e-05,
+      "loss": 1.291,
+      "theoretical_loss": 3.4242182823479883,
+      "tokens_seen": 2262237184
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.352110415663617e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.424202273217591,
+      "tokens_seen": 2262368256
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.351307976247794e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.4241862652743507,
+      "tokens_seen": 2262499328
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.35050553683197e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.424170258518111,
+      "tokens_seen": 2262630400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.349703097416146e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.4241542529487146,
+      "tokens_seen": 2262761472
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.348900658000321e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.4241382485660052,
+      "tokens_seen": 2262892544
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.348098218584497e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.4241222453698263,
+      "tokens_seen": 2263023616
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.347295779168673e-05,
+      "loss": 1.3106,
+      "theoretical_loss": 3.4241062433600207,
+      "tokens_seen": 2263154688
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.346493339752849e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4240902425364323,
+      "tokens_seen": 2263285760
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.345690900337025e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4240742428989037,
+      "tokens_seen": 2263416832
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.3448884609212e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.424058244447279,
+      "tokens_seen": 2263547904
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.344086021505376e-05,
+      "loss": 1.2841,
+      "theoretical_loss": 3.424042247181401,
+      "tokens_seen": 2263678976
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.343283582089553e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.4240262511011137,
+      "tokens_seen": 2263810048
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.342481142673728e-05,
+      "loss": 1.2908,
+      "theoretical_loss": 3.4240102562062606,
+      "tokens_seen": 2263941120
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.341678703257905e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4239942624966853,
+      "tokens_seen": 2264072192
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.34087626384208e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.4239782699722308,
+      "tokens_seen": 2264203264
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.340073824426257e-05,
+      "loss": 1.3193,
+      "theoretical_loss": 3.4239622786327413,
+      "tokens_seen": 2264334336
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.339271385010432e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.4239462884780596,
+      "tokens_seen": 2264465408
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.338468945594607e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.4239302995080303,
+      "tokens_seen": 2264596480
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.337666506178784e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.4239143117224966,
+      "tokens_seen": 2264727552
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.336864066762959e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.4238983251213018,
+      "tokens_seen": 2264858624
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.336061627347136e-05,
+      "loss": 1.3383,
+      "theoretical_loss": 3.4238823397042903,
+      "tokens_seen": 2264989696
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.335259187931311e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.423866355471306,
+      "tokens_seen": 2265120768
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.334456748515486e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4238503724221925,
+      "tokens_seen": 2265251840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.333654309099663e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.423834390556793,
+      "tokens_seen": 2265382912
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.332851869683838e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.423818409874952,
+      "tokens_seen": 2265513984
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.332049430268015e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.423802430376513,
+      "tokens_seen": 2265645056
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.33124699085219e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.42378645206132,
+      "tokens_seen": 2265776128
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.330444551436366e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.4237704749292175,
+      "tokens_seen": 2265907200
+    },
+    {
+      "epoch": 0.37,
+      "objective/train/docs_used": 761497,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2991373538970947,
+      "objective/train/theoretical_loss": 3.423754498980049,
+      "objective/train/tokens_used": 636497376,
+      "theoretical_loss": 3.423754498980049,
+      "tokens_seen": 2266038272
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.329642112020542e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.423754498980049,
+      "tokens_seen": 2266038272
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.328839672604718e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4237385242136584,
+      "tokens_seen": 2266169344
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.328037233188894e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.42372255062989,
+      "tokens_seen": 2266300416
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.32723479377307e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.4237065782285874,
+      "tokens_seen": 2266431488
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.326432354357246e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.423690607009595,
+      "tokens_seen": 2266562560
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.325629914941422e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.4236746369727573,
+      "tokens_seen": 2266693632
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.324827475525597e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.423658668117918,
+      "tokens_seen": 2266824704
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.324025036109774e-05,
+      "loss": 1.2747,
+      "theoretical_loss": 3.423642700444921,
+      "tokens_seen": 2266955776
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.323222596693949e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4236267339536113,
+      "tokens_seen": 2267086848
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.322420157278126e-05,
+      "loss": 1.2868,
+      "theoretical_loss": 3.423610768643832,
+      "tokens_seen": 2267217920
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.321617717862301e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.423594804515429,
+      "tokens_seen": 2267348992
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.320815278446478e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4235788415682453,
+      "tokens_seen": 2267480064
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.320012839030653e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.4235628798021254,
+      "tokens_seen": 2267611136
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.319210399614828e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.423546919216914,
+      "tokens_seen": 2267742208
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.318407960199005e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.4235309598124553,
+      "tokens_seen": 2267873280
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.31760552078318e-05,
+      "loss": 1.3307,
+      "theoretical_loss": 3.423515001588594,
+      "tokens_seen": 2268004352
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.316803081367357e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.423499044545174,
+      "tokens_seen": 2268135424
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.316000641951532e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.42348308868204,
+      "tokens_seen": 2268266496
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.315198202535709e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.4234671339990363,
+      "tokens_seen": 2268397568
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.314395763119884e-05,
+      "loss": 1.2847,
+      "theoretical_loss": 3.4234511804960084,
+      "tokens_seen": 2268528640
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 6.31359332370406e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.4234352281727998,
+      "tokens_seen": 2268659712
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.312790884288236e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.4234192770292555,
+      "tokens_seen": 2268790784
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.311988444872411e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.4234033270652198,
+      "tokens_seen": 2268921856
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.311186005456588e-05,
+      "loss": 1.1577,
+      "theoretical_loss": 3.423387378280538,
+      "tokens_seen": 2269052928
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.310383566040763e-05,
+      "loss": 1.1935,
+      "theoretical_loss": 3.4233714306750542,
+      "tokens_seen": 2269184000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.30958112662494e-05,
+      "loss": 1.31,
+      "theoretical_loss": 3.4233554842486136,
+      "tokens_seen": 2269315072
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.308778687209115e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.4233395390010606,
+      "tokens_seen": 2269446144
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.307976247793292e-05,
+      "loss": 1.1715,
+      "theoretical_loss": 3.42332359493224,
+      "tokens_seen": 2269577216
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.307173808377467e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4233076520419963,
+      "tokens_seen": 2269708288
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.306371368961643e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.423291710330175,
+      "tokens_seen": 2269839360
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.30556892954582e-05,
+      "loss": 1.2849,
+      "theoretical_loss": 3.42327576979662,
+      "tokens_seen": 2269970432
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.304766490129995e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.4232598304411774,
+      "tokens_seen": 2270101504
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.303964050714171e-05,
+      "loss": 1.3032,
+      "theoretical_loss": 3.4232438922636916,
+      "tokens_seen": 2270232576
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.303161611298347e-05,
+      "loss": 1.3252,
+      "theoretical_loss": 3.423227955264007,
+      "tokens_seen": 2270363648
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.302359171882523e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.4232120194419693,
+      "tokens_seen": 2270494720
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.301556732466699e-05,
+      "loss": 1.3107,
+      "theoretical_loss": 3.423196084797423,
+      "tokens_seen": 2270625792
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.300754293050874e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.4231801513302136,
+      "tokens_seen": 2270756864
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.29995185363505e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.423164219040186,
+      "tokens_seen": 2270887936
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.299149414219226e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4231482879271846,
+      "tokens_seen": 2271019008
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.298346974803403e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.4231323579910553,
+      "tokens_seen": 2271150080
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.297544535387578e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.4231164292316434,
+      "tokens_seen": 2271281152
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.296742095971755e-05,
+      "loss": 1.3145,
+      "theoretical_loss": 3.423100501648794,
+      "tokens_seen": 2271412224
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.29593965655593e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.4230845752423518,
+      "tokens_seen": 2271543296
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.295137217140105e-05,
+      "loss": 1.3743,
+      "theoretical_loss": 3.423068650012162,
+      "tokens_seen": 2271674368
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.294334777724282e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.4230527259580703,
+      "tokens_seen": 2271805440
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.293532338308457e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.4230368030799223,
+      "tokens_seen": 2271936512
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.292729898892634e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.423020881377562,
+      "tokens_seen": 2272067584
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.291927459476809e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.4230049608508364,
+      "tokens_seen": 2272198656
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.291125020060986e-05,
+      "loss": 1.321,
+      "theoretical_loss": 3.42298904149959,
+      "tokens_seen": 2272329728
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.290322580645161e-05,
+      "loss": 1.3185,
+      "theoretical_loss": 3.4229731233236684,
+      "tokens_seen": 2272460800
+    },
+    {
+      "epoch": 0.38,
+      "objective/train/docs_used": 763592,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2721900939941406,
+      "objective/train/theoretical_loss": 3.422957206322917,
+      "objective/train/tokens_used": 643050976,
+      "theoretical_loss": 3.422957206322917,
+      "tokens_seen": 2272591872
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.289520141229336e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.422957206322917,
+      "tokens_seen": 2272591872
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.288717701813513e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.4229412904971808,
+      "tokens_seen": 2272722944
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.287915262397688e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.4229253758463063,
+      "tokens_seen": 2272854016
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.287112822981865e-05,
+      "loss": 1.1726,
+      "theoretical_loss": 3.422909462370138,
+      "tokens_seen": 2272985088
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.28631038356604e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.4228935500685225,
+      "tokens_seen": 2273116160
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.285507944150217e-05,
+      "loss": 1.305,
+      "theoretical_loss": 3.422877638941304,
+      "tokens_seen": 2273247232
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.284705504734392e-05,
+      "loss": 1.3043,
+      "theoretical_loss": 3.42286172898833,
+      "tokens_seen": 2273378304
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.283903065318568e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4228458202094445,
+      "tokens_seen": 2273509376
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.283100625902744e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.422829912604494,
+      "tokens_seen": 2273640448
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.28229818648692e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4228140061733234,
+      "tokens_seen": 2273771520
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.281495747071096e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.4227981009157795,
+      "tokens_seen": 2273902592
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.280693307655272e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.4227821968317076,
+      "tokens_seen": 2274033664
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.279890868239448e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.4227662939209536,
+      "tokens_seen": 2274164736
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.279088428823624e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.422750392183363,
+      "tokens_seen": 2274295808
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.2782859894078e-05,
+      "loss": 1.3171,
+      "theoretical_loss": 3.422734491618782,
+      "tokens_seen": 2274426880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.277483549991976e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.422718592227056,
+      "tokens_seen": 2274557952
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.276681110576151e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4227026940080316,
+      "tokens_seen": 2274689024
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.275878671160328e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4226867969615538,
+      "tokens_seen": 2274820096
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.275076231744503e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.4226709010874696,
+      "tokens_seen": 2274951168
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.27427379232868e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.4226550063856247,
+      "tokens_seen": 2275082240
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.273471352912855e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.4226391128558644,
+      "tokens_seen": 2275213312
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.272668913497032e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.422623220498036,
+      "tokens_seen": 2275344384
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.271866474081207e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.4226073293119845,
+      "tokens_seen": 2275475456
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.271064034665382e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.4225914392975563,
+      "tokens_seen": 2275606528
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.270261595249559e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.422575550454598,
+      "tokens_seen": 2275737600
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.269459155833734e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.4225596627829553,
+      "tokens_seen": 2275868672
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.268656716417911e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4225437762824744,
+      "tokens_seen": 2275999744
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.267854277002086e-05,
+      "loss": 1.2079,
+      "theoretical_loss": 3.4225278909530017,
+      "tokens_seen": 2276130816
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.267051837586263e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.4225120067943835,
+      "tokens_seen": 2276261888
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.266249398170438e-05,
+      "loss": 1.3129,
+      "theoretical_loss": 3.422496123806466,
+      "tokens_seen": 2276392960
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.265446958754613e-05,
+      "loss": 1.2646,
+      "theoretical_loss": 3.4224802419890956,
+      "tokens_seen": 2276524032
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.26464451933879e-05,
+      "loss": 1.3211,
+      "theoretical_loss": 3.422464361342118,
+      "tokens_seen": 2276655104
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.263842079922965e-05,
+      "loss": 1.1754,
+      "theoretical_loss": 3.4224484818653806,
+      "tokens_seen": 2276786176
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.263039640507142e-05,
+      "loss": 1.3401,
+      "theoretical_loss": 3.42243260355873,
+      "tokens_seen": 2276917248
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.262237201091317e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.422416726422011,
+      "tokens_seen": 2277048320
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.261434761675494e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.4224008504550714,
+      "tokens_seen": 2277179392
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.26063232225967e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.4223849756577573,
+      "tokens_seen": 2277310464
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.259829882843845e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.422369102029915,
+      "tokens_seen": 2277441536
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.259027443428021e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.422353229571392,
+      "tokens_seen": 2277572608
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.258225004012197e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.422337358282033,
+      "tokens_seen": 2277703680
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.257422564596373e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.422321488161687,
+      "tokens_seen": 2277834752
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.256620125180549e-05,
+      "loss": 1.1887,
+      "theoretical_loss": 3.422305619210199,
+      "tokens_seen": 2277965824
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.255817685764725e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.4222897514274155,
+      "tokens_seen": 2278096896
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.255015246348901e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.4222738848131846,
+      "tokens_seen": 2278227968
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.254212806933076e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.4222580193673515,
+      "tokens_seen": 2278359040
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.253410367517253e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.4222421550897644,
+      "tokens_seen": 2278490112
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.252607928101428e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.422226291980269,
+      "tokens_seen": 2278621184
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.251805488685605e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.422210430038712,
+      "tokens_seen": 2278752256
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.25100304926978e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.422194569264941,
+      "tokens_seen": 2278883328
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.250200609853957e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.422178709658803,
+      "tokens_seen": 2279014400
+    },
+    {
+      "epoch": 0.38,
+      "objective/train/docs_used": 765727,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2659612894058228,
+      "objective/train/theoretical_loss": 3.422162851220144,
+      "objective/train/tokens_used": 649604576,
+      "theoretical_loss": 3.422162851220144,
+      "tokens_seen": 2279145472
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.249398170438132e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.422162851220144,
+      "tokens_seen": 2279145472
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.248595731022309e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.422146993948811,
+      "tokens_seen": 2279276544
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.247793291606484e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.422131137844652,
+      "tokens_seen": 2279407616
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.246990852190659e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.422115282907513,
+      "tokens_seen": 2279538688
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.246188412774836e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.422099429137241,
+      "tokens_seen": 2279669760
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.245385973359011e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.4220835765336837,
+      "tokens_seen": 2279800832
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.244583533943188e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.4220677250966878,
+      "tokens_seen": 2279931904
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.243781094527363e-05,
+      "loss": 1.3364,
+      "theoretical_loss": 3.4220518748261,
+      "tokens_seen": 2280062976
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.24297865511154e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.4220360257217686,
+      "tokens_seen": 2280194048
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.242176215695715e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.4220201777835397,
+      "tokens_seen": 2280325120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.24137377627989e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.4220043310112604,
+      "tokens_seen": 2280456192
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.240571336864067e-05,
+      "loss": 1.2747,
+      "theoretical_loss": 3.421988485404779,
+      "tokens_seen": 2280587264
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.239768897448242e-05,
+      "loss": 1.2928,
+      "theoretical_loss": 3.4219726409639413,
+      "tokens_seen": 2280718336
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.238966458032419e-05,
+      "loss": 1.234,
+      "theoretical_loss": 3.421956797688596,
+      "tokens_seen": 2280849408
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.238164018616594e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.421940955578589,
+      "tokens_seen": 2280980480
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.237361579200771e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4219251146337686,
+      "tokens_seen": 2281111552
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.236559139784946e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.421909274853982,
+      "tokens_seen": 2281242624
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.235756700369122e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.4218934362390763,
+      "tokens_seen": 2281373696
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.234954260953298e-05,
+      "loss": 1.1625,
+      "theoretical_loss": 3.421877598788899,
+      "tokens_seen": 2281504768
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.234151821537474e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.421861762503298,
+      "tokens_seen": 2281635840
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.23334938212165e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.4218459273821202,
+      "tokens_seen": 2281766912
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.232546942705826e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4218300934252133,
+      "tokens_seen": 2281897984
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.231744503290002e-05,
+      "loss": 1.1462,
+      "theoretical_loss": 3.421814260632425,
+      "tokens_seen": 2282029056
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.230942063874178e-05,
+      "loss": 1.1935,
+      "theoretical_loss": 3.4217984290036023,
+      "tokens_seen": 2282160128
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.230139624458353e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.4217825985385932,
+      "tokens_seen": 2282291200
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.22933718504253e-05,
+      "loss": 1.2096,
+      "theoretical_loss": 3.4217667692372458,
+      "tokens_seen": 2282422272
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.228534745626705e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.4217509410994067,
+      "tokens_seen": 2282553344
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.227732306210882e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4217351141249246,
+      "tokens_seen": 2282684416
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.226929866795057e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4217192883136462,
+      "tokens_seen": 2282815488
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.226127427379234e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.42170346366542,
+      "tokens_seen": 2282946560
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.225324987963409e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4216876401800933,
+      "tokens_seen": 2283077632
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.224522548547584e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.4216718178575145,
+      "tokens_seen": 2283208704
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.223720109131761e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.4216559966975306,
+      "tokens_seen": 2283339776
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.222917669715936e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.42164017669999,
+      "tokens_seen": 2283470848
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.222115230300113e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.4216243578647405,
+      "tokens_seen": 2283601920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.221312790884288e-05,
+      "loss": 1.1394,
+      "theoretical_loss": 3.42160854019163,
+      "tokens_seen": 2283732992
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.220510351468465e-05,
+      "loss": 1.1919,
+      "theoretical_loss": 3.421592723680506,
+      "tokens_seen": 2283864064
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.21970791205264e-05,
+      "loss": 1.1773,
+      "theoretical_loss": 3.4215769083312173,
+      "tokens_seen": 2283995136
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.218905472636816e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.4215610941436108,
+      "tokens_seen": 2284126208
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.218103033220992e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.4215452811175355,
+      "tokens_seen": 2284257280
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.217300593805168e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.421529469252839,
+      "tokens_seen": 2284388352
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.216498154389344e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.4215136585493697,
+      "tokens_seen": 2284519424
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.21569571497352e-05,
+      "loss": 1.1713,
+      "theoretical_loss": 3.421497849006975,
+      "tokens_seen": 2284650496
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.214893275557696e-05,
+      "loss": 1.1687,
+      "theoretical_loss": 3.4214820406255035,
+      "tokens_seen": 2284781568
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.214090836141871e-05,
+      "loss": 1.269,
+      "theoretical_loss": 3.4214662334048036,
+      "tokens_seen": 2284912640
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.213288396726048e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.421450427344723,
+      "tokens_seen": 2285043712
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 6.212485957310223e-05,
+      "loss": 1.1563,
+      "theoretical_loss": 3.42143462244511,
+      "tokens_seen": 2285174784
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.211683517894399e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.4214188187058134,
+      "tokens_seen": 2285305856
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.210881078478575e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.4214030161266806,
+      "tokens_seen": 2285436928
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.210078639062751e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.421387214707561,
+      "tokens_seen": 2285568000
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 767869,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1520464420318604,
+      "objective/train/theoretical_loss": 3.4213714144483016,
+      "objective/train/tokens_used": 656158176,
+      "theoretical_loss": 3.4213714144483016,
+      "tokens_seen": 2285699072
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.209276199646927e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.4213714144483016,
+      "tokens_seen": 2285699072
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.208473760231103e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.4213556153487517,
+      "tokens_seen": 2285830144
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.20767132081528e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.4213398174087595,
+      "tokens_seen": 2285961216
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.206868881399455e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.421324020628173,
+      "tokens_seen": 2286092288
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.20606644198363e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.4213082250068414,
+      "tokens_seen": 2286223360
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.205264002567807e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.4212924305446126,
+      "tokens_seen": 2286354432
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.204461563151982e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.4212766372413355,
+      "tokens_seen": 2286485504
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.203659123736159e-05,
+      "loss": 1.13,
+      "theoretical_loss": 3.421260845096858,
+      "tokens_seen": 2286616576
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.202856684320334e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.4212450541110293,
+      "tokens_seen": 2286747648
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.20205424490451e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.421229264283698,
+      "tokens_seen": 2286878720
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.201251805488686e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.421213475614712,
+      "tokens_seen": 2287009792
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.200449366072861e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.4211976881039208,
+      "tokens_seen": 2287140864
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.199646926657038e-05,
+      "loss": 1.3001,
+      "theoretical_loss": 3.421181901751172,
+      "tokens_seen": 2287271936
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.198844487241213e-05,
+      "loss": 1.3318,
+      "theoretical_loss": 3.4211661165563156,
+      "tokens_seen": 2287403008
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.19804204782539e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.4211503325191996,
+      "tokens_seen": 2287534080
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.197239608409565e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.421134549639673,
+      "tokens_seen": 2287665152
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.196437168993742e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.421118767917584,
+      "tokens_seen": 2287796224
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.195634729577917e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.421102987352782,
+      "tokens_seen": 2287927296
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.194832290162093e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4210872079451162,
+      "tokens_seen": 2288058368
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.194029850746269e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.4210714296944347,
+      "tokens_seen": 2288189440
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.193227411330445e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.4210556526005864,
+      "tokens_seen": 2288320512
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.192424971914621e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.421039876663421,
+      "tokens_seen": 2288451584
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.191622532498797e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.4210241018827867,
+      "tokens_seen": 2288582656
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.190820093082973e-05,
+      "loss": 1.1726,
+      "theoretical_loss": 3.4210083282585324,
+      "tokens_seen": 2288713728
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.190017653667148e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.420992555790508,
+      "tokens_seen": 2288844800
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.189215214251324e-05,
+      "loss": 1.291,
+      "theoretical_loss": 3.4209767844785617,
+      "tokens_seen": 2288975872
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.1884127748355e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.4209610143225433,
+      "tokens_seen": 2289106944
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.187610335419676e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.420945245322301,
+      "tokens_seen": 2289238016
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.186807896003852e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.4209294774776846,
+      "tokens_seen": 2289369088
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.186005456588028e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.420913710788543,
+      "tokens_seen": 2289500160
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.185203017172204e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.4208979452547252,
+      "tokens_seen": 2289631232
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.18440057775638e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.420882180876081,
+      "tokens_seen": 2289762304
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.183598138340556e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.4208664176524595,
+      "tokens_seen": 2289893376
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.182795698924732e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.4208506555837093,
+      "tokens_seen": 2290024448
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.181993259508907e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.4208348946696803,
+      "tokens_seen": 2290155520
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.181190820093084e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4208191349102215,
+      "tokens_seen": 2290286592
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.180388380677259e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.420803376305183,
+      "tokens_seen": 2290417664
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.179585941261436e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.420787618854413,
+      "tokens_seen": 2290548736
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.178783501845611e-05,
+      "loss": 1.2846,
+      "theoretical_loss": 3.420771862557762,
+      "tokens_seen": 2290679808
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.177981062429788e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4207561074150785,
+      "tokens_seen": 2290810880
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.177178623013963e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.420740353426213,
+      "tokens_seen": 2290941952
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.176376183598138e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.420724600591014,
+      "tokens_seen": 2291073024
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.175573744182315e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.4207088489093316,
+      "tokens_seen": 2291204096
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.17477130476649e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.4206930983810153,
+      "tokens_seen": 2291335168
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.173968865350667e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4206773490059144,
+      "tokens_seen": 2291466240
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.173166425934842e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4206616007838786,
+      "tokens_seen": 2291597312
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.172363986519019e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.420645853714758,
+      "tokens_seen": 2291728384
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.171561547103194e-05,
+      "loss": 1.2676,
+      "theoretical_loss": 3.420630107798402,
+      "tokens_seen": 2291859456
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.17075910768737e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.4206143630346597,
+      "tokens_seen": 2291990528
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.169956668271546e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.420598619423381,
+      "tokens_seen": 2292121600
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 770014,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2776598930358887,
+      "objective/train/theoretical_loss": 3.4205828769644167,
+      "objective/train/tokens_used": 662711776,
+      "theoretical_loss": 3.4205828769644167,
+      "tokens_seen": 2292252672
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.169154228855722e-05,
+      "loss": 1.3092,
+      "theoretical_loss": 3.4205828769644167,
+      "tokens_seen": 2292252672
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.168351789439898e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4205671356576155,
+      "tokens_seen": 2292383744
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.167549350024074e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.4205513955028275,
+      "tokens_seen": 2292514816
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.16674691060825e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4205356564999025,
+      "tokens_seen": 2292645888
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.165944471192425e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.420519918648691,
+      "tokens_seen": 2292776960
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.165142031776601e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.420504181949042,
+      "tokens_seen": 2292908032
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.164339592360777e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.420488446400805,
+      "tokens_seen": 2293039104
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.163537152944953e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.420472712003831,
+      "tokens_seen": 2293170176
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.16273471352913e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4204569787579704,
+      "tokens_seen": 2293301248
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.161932274113305e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.420441246663072,
+      "tokens_seen": 2293432320
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.161129834697481e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.4204255157189865,
+      "tokens_seen": 2293563392
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.160327395281657e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.420409785925563,
+      "tokens_seen": 2293694464
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.159524955865832e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.420394057282653,
+      "tokens_seen": 2293825536
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.158722516450009e-05,
+      "loss": 1.2054,
+      "theoretical_loss": 3.420378329790106,
+      "tokens_seen": 2293956608
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.157920077034184e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.4203626034477717,
+      "tokens_seen": 2294087680
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.157117637618361e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.420346878255501,
+      "tokens_seen": 2294218752
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.156315198202536e-05,
+      "loss": 1.1999,
+      "theoretical_loss": 3.4203311542131436,
+      "tokens_seen": 2294349824
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.155512758786713e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.42031543132055,
+      "tokens_seen": 2294480896
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.154710319370888e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.42029970957757,
+      "tokens_seen": 2294611968
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.153907879955065e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.4202839889840546,
+      "tokens_seen": 2294743040
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.15310544053924e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.4202682695398536,
+      "tokens_seen": 2294874112
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.152303001123415e-05,
+      "loss": 1.1811,
+      "theoretical_loss": 3.4202525512448174,
+      "tokens_seen": 2295005184
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.151500561707592e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.4202368340987963,
+      "tokens_seen": 2295136256
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.150698122291767e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.420221118101641,
+      "tokens_seen": 2295267328
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.149895682875944e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.420205403253202,
+      "tokens_seen": 2295398400
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.149093243460119e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4201896895533292,
+      "tokens_seen": 2295529472
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.148290804044296e-05,
+      "loss": 1.1605,
+      "theoretical_loss": 3.4201739770018733,
+      "tokens_seen": 2295660544
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.147488364628471e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.420158265598685,
+      "tokens_seen": 2295791616
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.146685925212647e-05,
+      "loss": 1.1659,
+      "theoretical_loss": 3.420142555343615,
+      "tokens_seen": 2295922688
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.145883485796823e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.4201268462365135,
+      "tokens_seen": 2296053760
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.145081046380999e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.420111138277231,
+      "tokens_seen": 2296184832
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.144278606965175e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.4200954314656182,
+      "tokens_seen": 2296315904
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.14347616754935e-05,
+      "loss": 1.3054,
+      "theoretical_loss": 3.420079725801526,
+      "tokens_seen": 2296446976
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.142673728133527e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.420064021284805,
+      "tokens_seen": 2296578048
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.141871288717702e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.4200483179153056,
+      "tokens_seen": 2296709120
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.141068849301878e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.420032615692879,
+      "tokens_seen": 2296840192
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.140266409886054e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.4200169146173756,
+      "tokens_seen": 2296971264
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.13946397047023e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.420001214688646,
+      "tokens_seen": 2297102336
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.138661531054406e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.419985515906542,
+      "tokens_seen": 2297233408
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.137859091638582e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.4199698182709133,
+      "tokens_seen": 2297364480
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.137056652222758e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.419954121781611,
+      "tokens_seen": 2297495552
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.136254212806934e-05,
+      "loss": 1.1842,
+      "theoretical_loss": 3.419938426438487,
+      "tokens_seen": 2297626624
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.135451773391109e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.4199227322413908,
+      "tokens_seen": 2297757696
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.134649333975286e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.419907039190174,
+      "tokens_seen": 2297888768
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.133846894559461e-05,
+      "loss": 1.2913,
+      "theoretical_loss": 3.419891347284688,
+      "tokens_seen": 2298019840
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.133044455143638e-05,
+      "loss": 1.1798,
+      "theoretical_loss": 3.419875656524783,
+      "tokens_seen": 2298150912
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.132242015727813e-05,
+      "loss": 1.1619,
+      "theoretical_loss": 3.4198599669103107,
+      "tokens_seen": 2298281984
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.13143957631199e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.4198442784411216,
+      "tokens_seen": 2298413056
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.130637136896165e-05,
+      "loss": 1.3163,
+      "theoretical_loss": 3.419828591117067,
+      "tokens_seen": 2298544128
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.12983469748034e-05,
+      "loss": 1.2955,
+      "theoretical_loss": 3.4198129049379986,
+      "tokens_seen": 2298675200
+    },
+    {
+      "epoch": 0.39,
+      "objective/train/docs_used": 772248,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2315409183502197,
+      "objective/train/theoretical_loss": 3.419797219903767,
+      "objective/train/tokens_used": 669265376,
+      "theoretical_loss": 3.419797219903767,
+      "tokens_seen": 2298806272
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.129032258064517e-05,
+      "loss": 1.1829,
+      "theoretical_loss": 3.419797219903767,
+      "tokens_seen": 2298806272
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.128229818648692e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4197815360142236,
+      "tokens_seen": 2298937344
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.127427379232869e-05,
+      "loss": 1.1962,
+      "theoretical_loss": 3.419765853269219,
+      "tokens_seen": 2299068416
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.126624939817044e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.419750171668605,
+      "tokens_seen": 2299199488
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.125822500401221e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.4197344912122327,
+      "tokens_seen": 2299330560
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.125020060985396e-05,
+      "loss": 1.1848,
+      "theoretical_loss": 3.419718811899954,
+      "tokens_seen": 2299461632
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.124217621569572e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.419703133731619,
+      "tokens_seen": 2299592704
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.123415182153748e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.41968745670708,
+      "tokens_seen": 2299723776
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.122612742737924e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.419671780826188,
+      "tokens_seen": 2299854848
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.1218103033221e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.419656106088794,
+      "tokens_seen": 2299985920
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.121007863906276e-05,
+      "loss": 1.1896,
+      "theoretical_loss": 3.4196404324947505,
+      "tokens_seen": 2300116992
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.120205424490452e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.4196247600439076,
+      "tokens_seen": 2300248064
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.119402985074628e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.4196090887361184,
+      "tokens_seen": 2300379136
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.118600545658804e-05,
+      "loss": 1.2406,
+      "theoretical_loss": 3.419593418571233,
+      "tokens_seen": 2300510208
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.11779810624298e-05,
+      "loss": 1.3162,
+      "theoretical_loss": 3.4195777495491035,
+      "tokens_seen": 2300641280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.116995666827155e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.4195620816695818,
+      "tokens_seen": 2300772352
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.116193227411331e-05,
+      "loss": 1.1797,
+      "theoretical_loss": 3.419546414932519,
+      "tokens_seen": 2300903424
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.115390787995507e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.419530749337767,
+      "tokens_seen": 2301034496
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.114588348579683e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.4195150848851767,
+      "tokens_seen": 2301165568
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.113785909163859e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.4194994215746006,
+      "tokens_seen": 2301296640
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.112983469748035e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.41948375940589,
+      "tokens_seen": 2301427712
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.112181030332211e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.4194680983788976,
+      "tokens_seen": 2301558784
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 6.111378590916386e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4194524384934737,
+      "tokens_seen": 2301689856
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.110576151500563e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.4194367797494714,
+      "tokens_seen": 2301820928
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.109773712084738e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4194211221467414,
+      "tokens_seen": 2301952000
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.108971272668915e-05,
+      "loss": 1.3299,
+      "theoretical_loss": 3.419405465685136,
+      "tokens_seen": 2302083072
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.10816883325309e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.419389810364507,
+      "tokens_seen": 2302214144
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.107366393837267e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.4193741561847064,
+      "tokens_seen": 2302345216
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.106563954421442e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.4193585031455864,
+      "tokens_seen": 2302476288
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.105761515005617e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.4193428512469985,
+      "tokens_seen": 2302607360
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.104959075589794e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.4193272004887953,
+      "tokens_seen": 2302738432
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.104156636173969e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.4193115508708276,
+      "tokens_seen": 2302869504
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.103354196758145e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.419295902392949,
+      "tokens_seen": 2303000576
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.1025517573423206e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.4192802550550097,
+      "tokens_seen": 2303131648
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.101749317926497e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.4192646088568637,
+      "tokens_seen": 2303262720
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.1009468785106726e-05,
+      "loss": 1.2522,
+      "theoretical_loss": 3.419248963798362,
+      "tokens_seen": 2303393792
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.100144439094848e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.419233319879357,
+      "tokens_seen": 2303524864
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0993419996790246e-05,
+      "loss": 1.2888,
+      "theoretical_loss": 3.4192176770997014,
+      "tokens_seen": 2303655936
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0985395602632e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4192020354592465,
+      "tokens_seen": 2303787008
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0977371208473766e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.4191863949578445,
+      "tokens_seen": 2303918080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.096934681431552e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.4191707555953483,
+      "tokens_seen": 2304049152
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0961322420157285e-05,
+      "loss": 1.1149,
+      "theoretical_loss": 3.4191551173716106,
+      "tokens_seen": 2304180224
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.095329802599904e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.4191394802864825,
+      "tokens_seen": 2304311296
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.094527363184079e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.419123844339817,
+      "tokens_seen": 2304442368
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.093724923768256e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.4191082095314664,
+      "tokens_seen": 2304573440
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.092922484352431e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.419092575861283,
+      "tokens_seen": 2304704512
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.092120044936608e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.4190769433291197,
+      "tokens_seen": 2304835584
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.091317605520783e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.4190613119348283,
+      "tokens_seen": 2304966656
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.09051516610496e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.4190456816782615,
+      "tokens_seen": 2305097728
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.089712726689135e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.419030052559272,
+      "tokens_seen": 2305228800
+    },
+    {
+      "debugging/Compilability": 0.0,
+      "debugging/distinct-1-grams": 0.6824214379946716,
+      "debugging/entropy-1-grams": 5.290924378994747,
+      "debugging/length": 550.125,
+      "debugging/num_segments": 16,
+      "debugging/score": 0.059994086139135025,
+      "debugging/score_std": 0.019143948003452343,
+      "epoch": 0.4,
+      "objective/train/docs_used": 774449,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.274184226989746,
+      "objective/train/theoretical_loss": 3.419014424577712,
+      "objective/train/tokens_used": 675818976,
+      "theoretical_loss": 3.419014424577712,
+      "tokens_seen": 2305359872
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.088910287273312e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.419014424577712,
+      "tokens_seen": 2305359872
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.088107847857487e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.418998797733434,
+      "tokens_seen": 2305490944
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0873054084416624e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.418983172026291,
+      "tokens_seen": 2305622016
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.086502969025839e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.418967547456136,
+      "tokens_seen": 2305753088
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0857005296100144e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.4189519240228203,
+      "tokens_seen": 2305884160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.084898090194191e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.4189363017261973,
+      "tokens_seen": 2306015232
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0840956507783664e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.4189206805661203,
+      "tokens_seen": 2306146304
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.083293211362543e-05,
+      "loss": 1.207,
+      "theoretical_loss": 3.4189050605424414,
+      "tokens_seen": 2306277376
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0824907719467183e-05,
+      "loss": 1.1871,
+      "theoretical_loss": 3.418889441655013,
+      "tokens_seen": 2306408448
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0816883325308937e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.418873823903689,
+      "tokens_seen": 2306539520
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.08088589311507e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.418858207288321,
+      "tokens_seen": 2306670592
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0800834536992456e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.4188425918087626,
+      "tokens_seen": 2306801664
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.079281014283422e-05,
+      "loss": 1.1843,
+      "theoretical_loss": 3.418826977464866,
+      "tokens_seen": 2306932736
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0784785748675976e-05,
+      "loss": 1.1586,
+      "theoretical_loss": 3.418811364256485,
+      "tokens_seen": 2307063808
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.077676135451774e-05,
+      "loss": 1.3036,
+      "theoretical_loss": 3.4187957521834713,
+      "tokens_seen": 2307194880
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0768736960359496e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.418780141245679,
+      "tokens_seen": 2307325952
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.076071256620125e-05,
+      "loss": 1.1537,
+      "theoretical_loss": 3.418764531442961,
+      "tokens_seen": 2307457024
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0752688172043016e-05,
+      "loss": 1.1706,
+      "theoretical_loss": 3.4187489227751695,
+      "tokens_seen": 2307588096
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.074466377788477e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.418733315242158,
+      "tokens_seen": 2307719168
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0736639383726536e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4187177088437797,
+      "tokens_seen": 2307850240
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.072861498956829e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.418702103579887,
+      "tokens_seen": 2307981312
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0720590595410055e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.418686499450334,
+      "tokens_seen": 2308112384
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.071256620125181e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.418670896454973,
+      "tokens_seen": 2308243456
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.070454180709356e-05,
+      "loss": 1.3268,
+      "theoretical_loss": 3.418655294593658,
+      "tokens_seen": 2308374528
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.069651741293533e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.418639693866241,
+      "tokens_seen": 2308505600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.068849301877708e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4186240942725767,
+      "tokens_seen": 2308636672
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.068046862461885e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.418608495812517,
+      "tokens_seen": 2308767744
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.06724442304606e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4185928984859157,
+      "tokens_seen": 2308898816
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.066441983630237e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4185773022926265,
+      "tokens_seen": 2309029888
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.065639544214412e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.4185617072325023,
+      "tokens_seen": 2309160960
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0648371047985874e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4185461133053963,
+      "tokens_seen": 2309292032
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.064034665382764e-05,
+      "loss": 1.2975,
+      "theoretical_loss": 3.418530520511162,
+      "tokens_seen": 2309423104
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0632322259669394e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.4185149288496532,
+      "tokens_seen": 2309554176
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.062429786551116e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4184993383207227,
+      "tokens_seen": 2309685248
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0616273471352914e-05,
+      "loss": 1.3021,
+      "theoretical_loss": 3.4184837489242246,
+      "tokens_seen": 2309816320
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.060824907719468e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.418468160660012,
+      "tokens_seen": 2309947392
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0600224683036434e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.4184525735279383,
+      "tokens_seen": 2310078464
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.05922002888782e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.4184369875278575,
+      "tokens_seen": 2310209536
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0584175894719954e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.418421402659623,
+      "tokens_seen": 2310340608
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.057615150056171e-05,
+      "loss": 1.2816,
+      "theoretical_loss": 3.418405818923088,
+      "tokens_seen": 2310471680
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.056812710640347e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.418390236318107,
+      "tokens_seen": 2310602752
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0560102712245226e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.4183746548445324,
+      "tokens_seen": 2310733824
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.055207831808699e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.418359074502219,
+      "tokens_seen": 2310864896
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0544053923928746e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.41834349529102,
+      "tokens_seen": 2310995968
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.053602952977051e-05,
+      "loss": 1.1896,
+      "theoretical_loss": 3.418327917210789,
+      "tokens_seen": 2311127040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0528005135612266e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.4183123402613806,
+      "tokens_seen": 2311258112
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.051998074145402e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.4182967644426476,
+      "tokens_seen": 2311389184
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0511956347295786e-05,
+      "loss": 1.1584,
+      "theoretical_loss": 3.4182811897544445,
+      "tokens_seen": 2311520256
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.050393195313754e-05,
+      "loss": 1.3178,
+      "theoretical_loss": 3.4182656161966247,
+      "tokens_seen": 2311651328
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0495907558979306e-05,
+      "loss": 1.1754,
+      "theoretical_loss": 3.418250043769042,
+      "tokens_seen": 2311782400
+    },
+    {
+      "epoch": 0.4,
+      "objective/train/docs_used": 776587,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2230169773101807,
+      "objective/train/theoretical_loss": 3.4182344724715508,
+      "objective/train/tokens_used": 682372576,
+      "theoretical_loss": 3.4182344724715508,
+      "tokens_seen": 2311913472
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.048788316482106e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.4182344724715508,
+      "tokens_seen": 2311913472
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0479858770662826e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.418218902304005,
+      "tokens_seen": 2312044544
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.047183437650458e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.418203333266258,
+      "tokens_seen": 2312175616
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.046380998234633e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.418187765358164,
+      "tokens_seen": 2312306688
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.04557855881881e-05,
+      "loss": 1.1845,
+      "theoretical_loss": 3.418172198579577,
+      "tokens_seen": 2312437760
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.044776119402985e-05,
+      "loss": 1.31,
+      "theoretical_loss": 3.4181566329303514,
+      "tokens_seen": 2312568832
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.043973679987162e-05,
+      "loss": 1.2669,
+      "theoretical_loss": 3.418141068410341,
+      "tokens_seen": 2312699904
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.043171240571337e-05,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4181255050194,
+      "tokens_seen": 2312830976
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.042368801155514e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.418109942757382,
+      "tokens_seen": 2312962048
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.041566361739689e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.418094381624142,
+      "tokens_seen": 2313093120
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0407639223238644e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4180788216195332,
+      "tokens_seen": 2313224192
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.039961482908041e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.4180632627434107,
+      "tokens_seen": 2313355264
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0391590434922164e-05,
+      "loss": 1.2241,
+      "theoretical_loss": 3.418047704995628,
+      "tokens_seen": 2313486336
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.038356604076393e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.41803214837604,
+      "tokens_seen": 2313617408
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0375541646605684e-05,
+      "loss": 1.2241,
+      "theoretical_loss": 3.4180165928845008,
+      "tokens_seen": 2313748480
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.036751725244745e-05,
+      "loss": 1.2169,
+      "theoretical_loss": 3.418001038520864,
+      "tokens_seen": 2313879552
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0359492858289204e-05,
+      "loss": 1.1916,
+      "theoretical_loss": 3.4179854852849854,
+      "tokens_seen": 2314010624
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.035146846413096e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.417969933176718,
+      "tokens_seen": 2314141696
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0343444069972724e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.4179543821959166,
+      "tokens_seen": 2314272768
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.033541967581448e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.4179388323424353,
+      "tokens_seen": 2314403840
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0327395281656243e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.4179232836161297,
+      "tokens_seen": 2314534912
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0319370887497997e-05,
+      "loss": 1.1914,
+      "theoretical_loss": 3.417907736016853,
+      "tokens_seen": 2314665984
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0311346493339756e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4178921895444603,
+      "tokens_seen": 2314797056
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0303322099181516e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.417876644198806,
+      "tokens_seen": 2314928128
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.029529770502327e-05,
+      "loss": 1.1537,
+      "theoretical_loss": 3.4178610999797447,
+      "tokens_seen": 2315059200
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0287273310865036e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.417845556887131,
+      "tokens_seen": 2315190272
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.027924891670679e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.4178300149208187,
+      "tokens_seen": 2315321344
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0271224522548556e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.417814474080664,
+      "tokens_seen": 2315452416
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.026320012839031e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.4177989343665205,
+      "tokens_seen": 2315583488
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.025517573423207e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.417783395778243,
+      "tokens_seen": 2315714560
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.024715134007383e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4177678583156865,
+      "tokens_seen": 2315845632
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.023912694591559e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.4177523219787056,
+      "tokens_seen": 2315976704
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.023110255175735e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4177367867671546,
+      "tokens_seen": 2316107776
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.02230781575991e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.417721252680889,
+      "tokens_seen": 2316238848
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.021505376344086e-05,
+      "loss": 1.1881,
+      "theoretical_loss": 3.4177057197197636,
+      "tokens_seen": 2316369920
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.020702936928262e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.4176901878836325,
+      "tokens_seen": 2316500992
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.019900497512438e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.417674657172351,
+      "tokens_seen": 2316632064
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.019098058096614e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.4176591275857744,
+      "tokens_seen": 2316763136
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.01829561868079e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.4176435991237577,
+      "tokens_seen": 2316894208
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.017493179264966e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.4176280717861545,
+      "tokens_seen": 2317025280
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0166907398491414e-05,
+      "loss": 1.1843,
+      "theoretical_loss": 3.4176125455728212,
+      "tokens_seen": 2317156352
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0158883004333174e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.4175970204836124,
+      "tokens_seen": 2317287424
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0150858610174934e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.417581496518383,
+      "tokens_seen": 2317418496
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0142834216016694e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.4175659736769877,
+      "tokens_seen": 2317549568
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0134809821858454e-05,
+      "loss": 1.1773,
+      "theoretical_loss": 3.4175504519592828,
+      "tokens_seen": 2317680640
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.0126785427700214e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.4175349313651218,
+      "tokens_seen": 2317811712
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.011876103354197e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4175194118943613,
+      "tokens_seen": 2317942784
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.011073663938373e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.4175038935468556,
+      "tokens_seen": 2318073856
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 6.010271224522549e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.41748837632246,
+      "tokens_seen": 2318204928
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.009468785106725e-05,
+      "loss": 1.3208,
+      "theoretical_loss": 3.41747286022103,
+      "tokens_seen": 2318336000
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 778698,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1850600242614746,
+      "objective/train/theoretical_loss": 3.4174573452424206,
+      "objective/train/tokens_used": 688926176,
+      "theoretical_loss": 3.4174573452424206,
+      "tokens_seen": 2318467072
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.008666345690901e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.4174573452424206,
+      "tokens_seen": 2318467072
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.007863906275077e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4174418313864874,
+      "tokens_seen": 2318598144
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.0070614668592527e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.417426318653085,
+      "tokens_seen": 2318729216
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.006259027443428e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.41741080704207,
+      "tokens_seen": 2318860288
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.005456588027604e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.4173952965532965,
+      "tokens_seen": 2318991360
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.00465414861178e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.4173797871866203,
+      "tokens_seen": 2319122432
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.003851709195956e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.417364278941897,
+      "tokens_seen": 2319253504
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.003049269780132e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.4173487718189817,
+      "tokens_seen": 2319384576
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.002246830364307e-05,
+      "loss": 1.1764,
+      "theoretical_loss": 3.4173332658177302,
+      "tokens_seen": 2319515648
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.001444390948484e-05,
+      "loss": 1.2101,
+      "theoretical_loss": 3.417317760937998,
+      "tokens_seen": 2319646720
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 6.000641951532659e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.4173022571796405,
+      "tokens_seen": 2319777792
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.999839512116835e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.417286754542513,
+      "tokens_seen": 2319908864
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.999037072701011e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.4172712530264713,
+      "tokens_seen": 2320039936
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.998234633285187e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.4172557526313714,
+      "tokens_seen": 2320171008
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.997432193869363e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4172402533570683,
+      "tokens_seen": 2320302080
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9966297544535385e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.4172247552034176,
+      "tokens_seen": 2320433152
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.995827315037715e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.4172092581702755,
+      "tokens_seen": 2320564224
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9950248756218905e-05,
+      "loss": 1.3032,
+      "theoretical_loss": 3.4171937622574973,
+      "tokens_seen": 2320695296
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.994222436206067e-05,
+      "loss": 1.2037,
+      "theoretical_loss": 3.417178267464939,
+      "tokens_seen": 2320826368
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9934199967902425e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.417162773792456,
+      "tokens_seen": 2320957440
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.992617557374418e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.4171472812399046,
+      "tokens_seen": 2321088512
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9918151179585944e-05,
+      "loss": 1.2847,
+      "theoretical_loss": 3.4171317898071405,
+      "tokens_seen": 2321219584
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.99101267854277e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.417116299494019,
+      "tokens_seen": 2321350656
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9902102391269464e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4171008103003966,
+      "tokens_seen": 2321481728
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.989407799711122e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.417085322226129,
+      "tokens_seen": 2321612800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9886053602952984e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.4170698352710716,
+      "tokens_seen": 2321743872
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.987802920879474e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.417054349435081,
+      "tokens_seen": 2321874944
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.987000481463649e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.4170388647180134,
+      "tokens_seen": 2322006016
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.986198042047826e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.417023381119724,
+      "tokens_seen": 2322137088
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.985395602632001e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.4170078986400694,
+      "tokens_seen": 2322268160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.984593163216178e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4169924172789052,
+      "tokens_seen": 2322399232
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.983790723800353e-05,
+      "loss": 1.2968,
+      "theoretical_loss": 3.416976937036088,
+      "tokens_seen": 2322530304
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.98298828438453e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4169614579114733,
+      "tokens_seen": 2322661376
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.982185844968705e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.4169459799049178,
+      "tokens_seen": 2322792448
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.98138340555288e-05,
+      "loss": 1.1629,
+      "theoretical_loss": 3.4169305030162773,
+      "tokens_seen": 2322923520
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.980580966137057e-05,
+      "loss": 1.2975,
+      "theoretical_loss": 3.4169150272454085,
+      "tokens_seen": 2323054592
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.979778526721232e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.4168995525921666,
+      "tokens_seen": 2323185664
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.978976087305409e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.416884079056409,
+      "tokens_seen": 2323316736
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.978173647889584e-05,
+      "loss": 1.1886,
+      "theoretical_loss": 3.4168686066379914,
+      "tokens_seen": 2323447808
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.977371208473761e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.41685313533677,
+      "tokens_seen": 2323578880
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.976568769057936e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.4168376651526007,
+      "tokens_seen": 2323709952
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9757663296421115e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.416822196085341,
+      "tokens_seen": 2323841024
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.974963890226288e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.416806728134847,
+      "tokens_seen": 2323972096
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9741614508104635e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.4167912613009737,
+      "tokens_seen": 2324103168
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.97335901139464e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.4167757955835794,
+      "tokens_seen": 2324234240
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9725565719788155e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.4167603309825196,
+      "tokens_seen": 2324365312
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.971754132562992e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.4167448674976506,
+      "tokens_seen": 2324496384
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9709516931471675e-05,
+      "loss": 1.1266,
+      "theoretical_loss": 3.4167294051288293,
+      "tokens_seen": 2324627456
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.970149253731343e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.416713943875912,
+      "tokens_seen": 2324758528
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9693468143155195e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.416698483738756,
+      "tokens_seen": 2324889600
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 780809,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1590662002563477,
+      "objective/train/theoretical_loss": 3.4166830247172166,
+      "objective/train/tokens_used": 695479776,
+      "theoretical_loss": 3.4166830247172166,
+      "tokens_seen": 2325020672
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.968544374899695e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.4166830247172166,
+      "tokens_seen": 2325020672
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9677419354838715e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.4166675668111517,
+      "tokens_seen": 2325151744
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.966939496068047e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.4166521100204172,
+      "tokens_seen": 2325282816
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9661370566522234e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.4166366543448694,
+      "tokens_seen": 2325413888
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.965334617236399e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.416621199784366,
+      "tokens_seen": 2325544960
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9645321778205754e-05,
+      "loss": 1.2823,
+      "theoretical_loss": 3.4166057463387633,
+      "tokens_seen": 2325676032
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.963729738404751e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.4165902940079174,
+      "tokens_seen": 2325807104
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.962927298988926e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.416574842791686,
+      "tokens_seen": 2325938176
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.962124859573103e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.4165593926899254,
+      "tokens_seen": 2326069248
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.961322420157278e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.4165439437024925,
+      "tokens_seen": 2326200320
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.960519980741455e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.4165284958292443,
+      "tokens_seen": 2326331392
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.95971754132563e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.4165130490700375,
+      "tokens_seen": 2326462464
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.958915101909807e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.416497603424729,
+      "tokens_seen": 2326593536
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.958112662493982e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.4164821588931757,
+      "tokens_seen": 2326724608
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.957310223078157e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.416466715475235,
+      "tokens_seen": 2326855680
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.956507783662334e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.416451273170763,
+      "tokens_seen": 2326986752
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.955705344246509e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.4164358319796175,
+      "tokens_seen": 2327117824
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.954902904830686e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.4164203919016556,
+      "tokens_seen": 2327248896
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.954100465414861e-05,
+      "loss": 1.1427,
+      "theoretical_loss": 3.4164049529367335,
+      "tokens_seen": 2327379968
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.953298025999038e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.416389515084709,
+      "tokens_seen": 2327511040
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.952495586583213e-05,
+      "loss": 1.1746,
+      "theoretical_loss": 3.416374078345439,
+      "tokens_seen": 2327642112
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9516931471673886e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.4163586427187806,
+      "tokens_seen": 2327773184
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.950890707751565e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4163432082045913,
+      "tokens_seen": 2327904256
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9500882683357405e-05,
+      "loss": 1.3341,
+      "theoretical_loss": 3.4163277748027276,
+      "tokens_seen": 2328035328
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.949285828919917e-05,
+      "loss": 1.2819,
+      "theoretical_loss": 3.4163123425130473,
+      "tokens_seen": 2328166400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9484833895040925e-05,
+      "loss": 1.2124,
+      "theoretical_loss": 3.4162969113354076,
+      "tokens_seen": 2328297472
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.947680950088269e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.4162814812696656,
+      "tokens_seen": 2328428544
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9468785106724445e-05,
+      "loss": 1.1787,
+      "theoretical_loss": 3.4162660523156787,
+      "tokens_seen": 2328559616
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.94607607125662e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.416250624473304,
+      "tokens_seen": 2328690688
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9452736318407965e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.4162351977423993,
+      "tokens_seen": 2328821760
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.944471192424972e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.4162197721228216,
+      "tokens_seen": 2328952832
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9436687530091485e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.4162043476144284,
+      "tokens_seen": 2329083904
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.942866313593324e-05,
+      "loss": 1.2928,
+      "theoretical_loss": 3.416188924217077,
+      "tokens_seen": 2329214976
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9420638741775004e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.416173501930625,
+      "tokens_seen": 2329346048
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.941261434761676e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.41615808075493,
+      "tokens_seen": 2329477120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.940458995345851e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.4161426606898493,
+      "tokens_seen": 2329608192
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.939656555930028e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.416127241735241,
+      "tokens_seen": 2329739264
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.938854116514203e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.4161118238909616,
+      "tokens_seen": 2329870336
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.93805167709838e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4160964071568696,
+      "tokens_seen": 2330001408
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.937249237682555e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.416080991532822,
+      "tokens_seen": 2330132480
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.936446798266732e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.416065577018677,
+      "tokens_seen": 2330263552
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.935644358850907e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.4160501636142913,
+      "tokens_seen": 2330394624
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.934841919435082e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.4160347513195237,
+      "tokens_seen": 2330525696
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.934039480019259e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.416019340134232,
+      "tokens_seen": 2330656768
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.933237040603434e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.416003930058273,
+      "tokens_seen": 2330787840
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.932434601187611e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.4159885210915046,
+      "tokens_seen": 2330918912
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.931632161771786e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.415973113233785,
+      "tokens_seen": 2331049984
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.930829722355963e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.4159577064849724,
+      "tokens_seen": 2331181056
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.930027282940138e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.415942300844924,
+      "tokens_seen": 2331312128
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.929224843524315e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.415926896313498,
+      "tokens_seen": 2331443200
+    },
+    {
+      "epoch": 0.41,
+      "objective/train/docs_used": 782999,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2416090965270996,
+      "objective/train/theoretical_loss": 3.415911492890552,
+      "objective/train/tokens_used": 702033376,
+      "theoretical_loss": 3.415911492890552,
+      "tokens_seen": 2331574272
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.92842240410849e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.415911492890552,
+      "tokens_seen": 2331574272
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9276199646926656e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.4158960905759437,
+      "tokens_seen": 2331705344
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.926817525276842e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4158806893695317,
+      "tokens_seen": 2331836416
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9260150858610175e-05,
+      "loss": 1.3078,
+      "theoretical_loss": 3.415865289271174,
+      "tokens_seen": 2331967488
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.925212646445194e-05,
+      "loss": 1.2963,
+      "theoretical_loss": 3.415849890280728,
+      "tokens_seen": 2332098560
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9244102070293695e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.4158344923980524,
+      "tokens_seen": 2332229632
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.923607767613546e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.4158190956230046,
+      "tokens_seen": 2332360704
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9228053281977215e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.4158036999554433,
+      "tokens_seen": 2332491776
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.922002888781897e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.415788305395226,
+      "tokens_seen": 2332622848
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9212004493660735e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.415772911942211,
+      "tokens_seen": 2332753920
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.920398009950249e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.415757519596257,
+      "tokens_seen": 2332884992
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9195955705344255e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.4157421283572216,
+      "tokens_seen": 2333016064
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.918793131118601e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.4157267382249636,
+      "tokens_seen": 2333147136
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.9179906917027774e-05,
+      "loss": 1.2105,
+      "theoretical_loss": 3.4157113491993405,
+      "tokens_seen": 2333278208
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.917188252286953e-05,
+      "loss": 1.1673,
+      "theoretical_loss": 3.415695961280211,
+      "tokens_seen": 2333409280
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.916385812871128e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.4156805744674337,
+      "tokens_seen": 2333540352
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.915583373455305e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.415665188760866,
+      "tokens_seen": 2333671424
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.91478093403948e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.415649804160367,
+      "tokens_seen": 2333802496
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.913978494623657e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.4156344206657945,
+      "tokens_seen": 2333933568
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.913176055207832e-05,
+      "loss": 1.1504,
+      "theoretical_loss": 3.4156190382770077,
+      "tokens_seen": 2334064640
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.912373615792009e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4156036569938646,
+      "tokens_seen": 2334195712
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.911571176376184e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.415588276816224,
+      "tokens_seen": 2334326784
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.910768736960359e-05,
+      "loss": 1.1432,
+      "theoretical_loss": 3.4155728977439432,
+      "tokens_seen": 2334457856
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.909966297544536e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.415557519776882,
+      "tokens_seen": 2334588928
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.909163858128711e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.4155421429148984,
+      "tokens_seen": 2334720000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.908361418712888e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.415526767157851,
+      "tokens_seen": 2334851072
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.907558979297063e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.415511392505598,
+      "tokens_seen": 2334982144
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.90675653988124e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.415496018957999,
+      "tokens_seen": 2335113216
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.905954100465415e-05,
+      "loss": 1.1897,
+      "theoretical_loss": 3.415480646514912,
+      "tokens_seen": 2335244288
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.9051516610495906e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.4154652751761954,
+      "tokens_seen": 2335375360
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.904349221633767e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.415449904941708,
+      "tokens_seen": 2335506432
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.9035467822179426e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4154345358113094,
+      "tokens_seen": 2335637504
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.902744342802119e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.415419167784857,
+      "tokens_seen": 2335768576
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.9019419033862945e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.4154038008622103,
+      "tokens_seen": 2335899648
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.901139463970471e-05,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4153884350432278,
+      "tokens_seen": 2336030720
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.9003370245546465e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.415373070327769,
+      "tokens_seen": 2336161792
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8995345851388225e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.415357706715692,
+      "tokens_seen": 2336292864
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8987321457229985e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.415342344206856,
+      "tokens_seen": 2336423936
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.897929706307174e-05,
+      "loss": 1.2785,
+      "theoretical_loss": 3.415326982801119,
+      "tokens_seen": 2336555008
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8971272668913505e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.4153116224983417,
+      "tokens_seen": 2336686080
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.896324827475526e-05,
+      "loss": 1.1748,
+      "theoretical_loss": 3.4152962632983814,
+      "tokens_seen": 2336817152
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8955223880597025e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.415280905201098,
+      "tokens_seen": 2336948224
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.894719948643878e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.4152655482063503,
+      "tokens_seen": 2337079296
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.893917509228054e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.415250192313997,
+      "tokens_seen": 2337210368
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.89311506981223e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.415234837523898,
+      "tokens_seen": 2337341440
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.892312630396405e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.4152194838359113,
+      "tokens_seen": 2337472512
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.891510190980582e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.4152041312498964,
+      "tokens_seen": 2337603584
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.890707751564757e-05,
+      "loss": 1.1677,
+      "theoretical_loss": 3.4151887797657126,
+      "tokens_seen": 2337734656
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.889905312148933e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.415173429383219,
+      "tokens_seen": 2337865728
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.889102872733109e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.415158080102275,
+      "tokens_seen": 2337996800
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 785151,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2727793455123901,
+      "objective/train/theoretical_loss": 3.4151427319227388,
+      "objective/train/tokens_used": 708586976,
+      "theoretical_loss": 3.4151427319227388,
+      "tokens_seen": 2338127872
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.888300433317285e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.4151427319227388,
+      "tokens_seen": 2338127872
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.887497993901461e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.415127384844471,
+      "tokens_seen": 2338258944
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.886695554485636e-05,
+      "loss": 1.1555,
+      "theoretical_loss": 3.41511203886733,
+      "tokens_seen": 2338390016
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.885893115069813e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.4150966939911753,
+      "tokens_seen": 2338521088
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.885090675653988e-05,
+      "loss": 1.1306,
+      "theoretical_loss": 3.4150813502158663,
+      "tokens_seen": 2338652160
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.884288236238164e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.415066007541262,
+      "tokens_seen": 2338783232
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.88348579682234e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.4150506659672226,
+      "tokens_seen": 2338914304
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.882683357406516e-05,
+      "loss": 1.2841,
+      "theoretical_loss": 3.4150353254936063,
+      "tokens_seen": 2339045376
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.881880917990692e-05,
+      "loss": 1.1999,
+      "theoretical_loss": 3.4150199861202735,
+      "tokens_seen": 2339176448
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8810784785748676e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.415004647847083,
+      "tokens_seen": 2339307520
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8802760391590436e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.414989310673894,
+      "tokens_seen": 2339438592
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8794735997432196e-05,
+      "loss": 1.2934,
+      "theoretical_loss": 3.414973974600567,
+      "tokens_seen": 2339569664
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8786711603273956e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.414958639626961,
+      "tokens_seen": 2339700736
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8778687209115716e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.414943305752936,
+      "tokens_seen": 2339831808
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8770662814957475e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.4149279729783504,
+      "tokens_seen": 2339962880
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8762638420799235e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.4149126413030646,
+      "tokens_seen": 2340093952
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.875461402664099e-05,
+      "loss": 1.2488,
+      "theoretical_loss": 3.4148973107269383,
+      "tokens_seen": 2340225024
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.874658963248275e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.414881981249831,
+      "tokens_seen": 2340356096
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.873856523832451e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.414866652871602,
+      "tokens_seen": 2340487168
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.873054084416627e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.414851325592112,
+      "tokens_seen": 2340618240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.872251645000803e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.4148359994112196,
+      "tokens_seen": 2340749312
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.871449205584979e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.414820674328785,
+      "tokens_seen": 2340880384
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.870646766169154e-05,
+      "loss": 1.1579,
+      "theoretical_loss": 3.4148053503446683,
+      "tokens_seen": 2341011456
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.869844326753331e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.414790027458728,
+      "tokens_seen": 2341142528
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.869041887337506e-05,
+      "loss": 1.3118,
+      "theoretical_loss": 3.414774705670826,
+      "tokens_seen": 2341273600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.868239447921682e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.4147593849808207,
+      "tokens_seen": 2341404672
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.867437008505858e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.414744065388572,
+      "tokens_seen": 2341535744
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8666345690900334e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4147287468939407,
+      "tokens_seen": 2341666816
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.86583212967421e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.4147134294967856,
+      "tokens_seen": 2341797888
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8650296902583854e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.414698113196967,
+      "tokens_seen": 2341928960
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.864227250842562e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4146827979943457,
+      "tokens_seen": 2342060032
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8634248114267374e-05,
+      "loss": 1.217,
+      "theoretical_loss": 3.4146674838887807,
+      "tokens_seen": 2342191104
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8626223720109133e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.4146521708801325,
+      "tokens_seen": 2342322176
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.861819932595089e-05,
+      "loss": 1.1887,
+      "theoretical_loss": 3.414636858968261,
+      "tokens_seen": 2342453248
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8610174931792646e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.4146215481530264,
+      "tokens_seen": 2342584320
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.860215053763441e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.4146062384342892,
+      "tokens_seen": 2342715392
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8594126143476166e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4145909298119084,
+      "tokens_seen": 2342846464
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.858610174931793e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.4145756222857453,
+      "tokens_seen": 2342977536
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8578077355159686e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4145603158556592,
+      "tokens_seen": 2343108608
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.857005296100144e-05,
+      "loss": 1.2113,
+      "theoretical_loss": 3.414545010521511,
+      "tokens_seen": 2343239680
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8562028566843206e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.4145297062831608,
+      "tokens_seen": 2343370752
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.855400417268496e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.4145144031404686,
+      "tokens_seen": 2343501824
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8545979778526726e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.414499101093295,
+      "tokens_seen": 2343632896
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.853795538436848e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.4144838001414994,
+      "tokens_seen": 2343763968
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8529930990210246e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.4144685002849435,
+      "tokens_seen": 2343895040
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8521906596052e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.414453201523487,
+      "tokens_seen": 2344026112
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.851388220189375e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.4144379038569905,
+      "tokens_seen": 2344157184
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.850585780773552e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4144226072853137,
+      "tokens_seen": 2344288256
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.849783341357727e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4144073118083176,
+      "tokens_seen": 2344419328
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.848980901941904e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.414392017425863,
+      "tokens_seen": 2344550400
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 787449,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2394485473632812,
+      "objective/train/theoretical_loss": 3.41437672413781,
+      "objective/train/tokens_used": 715140576,
+      "theoretical_loss": 3.41437672413781,
+      "tokens_seen": 2344681472
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.848178462526079e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.41437672413781,
+      "tokens_seen": 2344681472
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.847376023110256e-05,
+      "loss": 1.2846,
+      "theoretical_loss": 3.414361431944019,
+      "tokens_seen": 2344812544
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.846573583694431e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4143461408443505,
+      "tokens_seen": 2344943616
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8457711442786064e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.414330850838666,
+      "tokens_seen": 2345074688
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.844968704862783e-05,
+      "loss": 1.3421,
+      "theoretical_loss": 3.4143155619268244,
+      "tokens_seen": 2345205760
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8441662654469584e-05,
+      "loss": 1.3056,
+      "theoretical_loss": 3.4143002741086876,
+      "tokens_seen": 2345336832
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.843363826031135e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.414284987384116,
+      "tokens_seen": 2345467904
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8425613866153104e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.4142697017529704,
+      "tokens_seen": 2345598976
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.841758947199487e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.414254417215111,
+      "tokens_seen": 2345730048
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8409565077836624e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.414239133770399,
+      "tokens_seen": 2345861120
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.840154068367838e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.414223851418695,
+      "tokens_seen": 2345992192
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8393516289520144e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4142085701598592,
+      "tokens_seen": 2346123264
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.83854918953619e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.4141932899937535,
+      "tokens_seen": 2346254336
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8377467501203663e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4141780109202378,
+      "tokens_seen": 2346385408
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8369443107045417e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4141627329391735,
+      "tokens_seen": 2346516480
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.836141871288718e-05,
+      "loss": 1.331,
+      "theoretical_loss": 3.4141474560504212,
+      "tokens_seen": 2346647552
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8353394318728936e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.4141321802538416,
+      "tokens_seen": 2346778624
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.83453699245707e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.414116905549296,
+      "tokens_seen": 2346909696
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8337345530412456e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.4141016319366457,
+      "tokens_seen": 2347040768
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.832932113625421e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.4140863594157507,
+      "tokens_seen": 2347171840
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8321296742095976e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4140710879864726,
+      "tokens_seen": 2347302912
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.831327234793773e-05,
+      "loss": 1.1845,
+      "theoretical_loss": 3.4140558176486726,
+      "tokens_seen": 2347433984
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8305247953779496e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.414040548402211,
+      "tokens_seen": 2347565056
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.829722355962125e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4140252802469497,
+      "tokens_seen": 2347696128
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8289199165463016e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.4140100131827493,
+      "tokens_seen": 2347827200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.828117477130477e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.4139947472094714,
+      "tokens_seen": 2347958272
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.827315037714652e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.413979482326977,
+      "tokens_seen": 2348089344
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.826512598298829e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.413964218535127,
+      "tokens_seen": 2348220416
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.825710158883004e-05,
+      "loss": 1.2653,
+      "theoretical_loss": 3.4139489558337823,
+      "tokens_seen": 2348351488
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.824907719467181e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4139336942228047,
+      "tokens_seen": 2348482560
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.824105280051356e-05,
+      "loss": 1.1694,
+      "theoretical_loss": 3.4139184337020554,
+      "tokens_seen": 2348613632
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.823302840635533e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.413903174271396,
+      "tokens_seen": 2348744704
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.822500401219708e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.4138879159306867,
+      "tokens_seen": 2348875776
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8216979618038834e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4138726586797903,
+      "tokens_seen": 2349006848
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.82089552238806e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.413857402518567,
+      "tokens_seen": 2349137920
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8200930829722354e-05,
+      "loss": 1.3214,
+      "theoretical_loss": 3.4138421474468785,
+      "tokens_seen": 2349268992
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.819290643556412e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.4138268934645866,
+      "tokens_seen": 2349400064
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8184882041405874e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.4138116405715526,
+      "tokens_seen": 2349531136
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.817685764724764e-05,
+      "loss": 1.2875,
+      "theoretical_loss": 3.4137963887676372,
+      "tokens_seen": 2349662208
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8168833253089394e-05,
+      "loss": 1.1496,
+      "theoretical_loss": 3.413781138052703,
+      "tokens_seen": 2349793280
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.816080885893115e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.4137658884266107,
+      "tokens_seen": 2349924352
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8152784464772914e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.4137506398892223,
+      "tokens_seen": 2350055424
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.814476007061467e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.413735392440399,
+      "tokens_seen": 2350186496
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8136735676456433e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.413720146080003,
+      "tokens_seen": 2350317568
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.812871128229819e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.4137049008078946,
+      "tokens_seen": 2350448640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.812068688813995e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.4136896566239368,
+      "tokens_seen": 2350579712
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8112662493981706e-05,
+      "loss": 1.1975,
+      "theoretical_loss": 3.413674413527991,
+      "tokens_seen": 2350710784
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.810463809982346e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.4136591715199183,
+      "tokens_seen": 2350841856
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.8096613705665226e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.413643930599581,
+      "tokens_seen": 2350972928
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.808858931150698e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.4136286907668403,
+      "tokens_seen": 2351104000
+    },
+    {
+      "epoch": 0.42,
+      "objective/train/docs_used": 789531,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3354337215423584,
+      "objective/train/theoretical_loss": 3.4136134520215586,
+      "objective/train/tokens_used": 721694176,
+      "theoretical_loss": 3.4136134520215586,
+      "tokens_seen": 2351235072
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.8080564917348746e-05,
+      "loss": 1.2913,
+      "theoretical_loss": 3.4136134520215586,
+      "tokens_seen": 2351235072
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.80725405231905e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.4135982143635974,
+      "tokens_seen": 2351366144
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.8064516129032266e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.413582977792818,
+      "tokens_seen": 2351497216
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.805649173487402e-05,
+      "loss": 1.1909,
+      "theoretical_loss": 3.413567742309083,
+      "tokens_seen": 2351628288
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.8048467340715786e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.413552507912254,
+      "tokens_seen": 2351759360
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.804044294655754e-05,
+      "loss": 1.1451,
+      "theoretical_loss": 3.4135372746021932,
+      "tokens_seen": 2351890432
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.803241855239929e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.413522042378762,
+      "tokens_seen": 2352021504
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.802439415824106e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.413506811241822,
+      "tokens_seen": 2352152576
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.801636976408281e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.413491581191236,
+      "tokens_seen": 2352283648
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.800834536992458e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.413476352226866,
+      "tokens_seen": 2352414720
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.800032097576633e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.4134611243485735,
+      "tokens_seen": 2352545792
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.79922965816081e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.4134458975562207,
+      "tokens_seen": 2352676864
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.798427218744985e-05,
+      "loss": 1.3286,
+      "theoretical_loss": 3.4134306718496696,
+      "tokens_seen": 2352807936
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7976247793291605e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.4134154472287825,
+      "tokens_seen": 2352939008
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.796822339913337e-05,
+      "loss": 1.3049,
+      "theoretical_loss": 3.4134002236934213,
+      "tokens_seen": 2353070080
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7960199004975124e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.413385001243448,
+      "tokens_seen": 2353201152
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.795217461081689e-05,
+      "loss": 1.3067,
+      "theoretical_loss": 3.4133697798787255,
+      "tokens_seen": 2353332224
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7944150216658644e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.4133545595991155,
+      "tokens_seen": 2353463296
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.793612582250041e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.41333934040448,
+      "tokens_seen": 2353594368
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7928101428342164e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.4133241222946813,
+      "tokens_seen": 2353725440
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.792007703418392e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.4133089052695818,
+      "tokens_seen": 2353856512
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7912052640025684e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.413293689329044,
+      "tokens_seen": 2353987584
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.790402824586744e-05,
+      "loss": 1.3364,
+      "theoretical_loss": 3.4132784744729303,
+      "tokens_seen": 2354118656
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7896003851709204e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.413263260701102,
+      "tokens_seen": 2354249728
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.788797945755096e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.4132480480134233,
+      "tokens_seen": 2354380800
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.787995506339272e-05,
+      "loss": 1.3176,
+      "theoretical_loss": 3.4132328364097546,
+      "tokens_seen": 2354511872
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7871930669234477e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.41321762588996,
+      "tokens_seen": 2354642944
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.786390627507623e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.4132024164539008,
+      "tokens_seen": 2354774016
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7855881880917996e-05,
+      "loss": 1.3076,
+      "theoretical_loss": 3.41318720810144,
+      "tokens_seen": 2354905088
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.784785748675975e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.4131720008324398,
+      "tokens_seen": 2355036160
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7839833092601516e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.413156794646763,
+      "tokens_seen": 2355167232
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.783180869844327e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.413141589544272,
+      "tokens_seen": 2355298304
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7823784304285036e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.4131263855248295,
+      "tokens_seen": 2355429376
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.781575991012679e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.4131111825882976,
+      "tokens_seen": 2355560448
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.780773551596854e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4130959807345396,
+      "tokens_seen": 2355691520
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.779971112181031e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4130807799634177,
+      "tokens_seen": 2355822592
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.779168672765206e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.413065580274795,
+      "tokens_seen": 2355953664
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.778366233349383e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.413050381668534,
+      "tokens_seen": 2356084736
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.777563793933558e-05,
+      "loss": 1.3038,
+      "theoretical_loss": 3.413035184144497,
+      "tokens_seen": 2356215808
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.776761354517735e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.4130199877025467,
+      "tokens_seen": 2356346880
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.77595891510191e-05,
+      "loss": 1.317,
+      "theoretical_loss": 3.413004792342547,
+      "tokens_seen": 2356477952
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.775156475686087e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.4129895980643594,
+      "tokens_seen": 2356609024
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.774354036270262e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.4129744048678474,
+      "tokens_seen": 2356740096
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7735515968544375e-05,
+      "loss": 1.2847,
+      "theoretical_loss": 3.412959212752874,
+      "tokens_seen": 2356871168
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.772749157438614e-05,
+      "loss": 1.2101,
+      "theoretical_loss": 3.4129440217193014,
+      "tokens_seen": 2357002240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7719467180227894e-05,
+      "loss": 1.1751,
+      "theoretical_loss": 3.412928831766993,
+      "tokens_seen": 2357133312
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.771144278606966e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.412913642895812,
+      "tokens_seen": 2357264384
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7703418391911414e-05,
+      "loss": 1.2097,
+      "theoretical_loss": 3.41289845510562,
+      "tokens_seen": 2357395456
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.769539399775318e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.4128832683962815,
+      "tokens_seen": 2357526528
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7687369603594934e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.412868082767659,
+      "tokens_seen": 2357657600
+    },
+    {
+      "epoch": 0.43,
+      "objective/train/docs_used": 791745,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2289501428604126,
+      "objective/train/theoretical_loss": 3.412852898219615,
+      "objective/train/tokens_used": 728247776,
+      "theoretical_loss": 3.412852898219615,
+      "tokens_seen": 2357788672
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.767934520943669e-05,
+      "loss": 1.3085,
+      "theoretical_loss": 3.412852898219615,
+      "tokens_seen": 2357788672
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7671320815278454e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.412837714752013,
+      "tokens_seen": 2357919744
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.766329642112021e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.412822532364716,
+      "tokens_seen": 2358050816
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7655272026961974e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.4128073510575874,
+      "tokens_seen": 2358181888
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.764724763280373e-05,
+      "loss": 1.2833,
+      "theoretical_loss": 3.41279217083049,
+      "tokens_seen": 2358312960
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.763922323864549e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.4127769916832866,
+      "tokens_seen": 2358444032
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7631198844487247e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.412761813615841,
+      "tokens_seen": 2358575104
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7623174450329e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4127466366280164,
+      "tokens_seen": 2358706176
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7615150056170766e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.4127314607196753,
+      "tokens_seen": 2358837248
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.760712566201252e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.412716285890682,
+      "tokens_seen": 2358968320
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7599101267854286e-05,
+      "loss": 1.1795,
+      "theoretical_loss": 3.4127011121408986,
+      "tokens_seen": 2359099392
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.759107687369604e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.412685939470189,
+      "tokens_seen": 2359230464
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.75830524795378e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.4126707678784167,
+      "tokens_seen": 2359361536
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.757502808537956e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.4126555973654447,
+      "tokens_seen": 2359492608
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.756700369122131e-05,
+      "loss": 1.2872,
+      "theoretical_loss": 3.4126404279311364,
+      "tokens_seen": 2359623680
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.755897929706308e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.4126252595753552,
+      "tokens_seen": 2359754752
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.755095490290483e-05,
+      "loss": 1.2823,
+      "theoretical_loss": 3.412610092297965,
+      "tokens_seen": 2359885824
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.754293050874659e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.412594926098829,
+      "tokens_seen": 2360016896
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.753490611458835e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.4125797609778097,
+      "tokens_seen": 2360147968
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.752688172043011e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4125645969347724,
+      "tokens_seen": 2360279040
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.751885732627187e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.4125494339695788,
+      "tokens_seen": 2360410112
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7510832932113625e-05,
+      "loss": 1.2072,
+      "theoretical_loss": 3.4125342720820937,
+      "tokens_seen": 2360541184
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.750280853795539e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.41251911127218,
+      "tokens_seen": 2360672256
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7494784143797145e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4125039515397018,
+      "tokens_seen": 2360803328
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7486759749638905e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.4124887928845222,
+      "tokens_seen": 2360934400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7478735355480664e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.412473635306505,
+      "tokens_seen": 2361065472
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7470710961322424e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4124584788055143,
+      "tokens_seen": 2361196544
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7462686567164184e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4124433233814133,
+      "tokens_seen": 2361327616
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.745466217300594e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.4124281690340657,
+      "tokens_seen": 2361458688
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.74466377788477e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.4124130157633354,
+      "tokens_seen": 2361589760
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.743861338468946e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.412397863569087,
+      "tokens_seen": 2361720832
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.743058899053122e-05,
+      "loss": 1.1892,
+      "theoretical_loss": 3.4123827124511825,
+      "tokens_seen": 2361851904
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.742256459637298e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.4123675624094867,
+      "tokens_seen": 2361982976
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.741454020221474e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.4123524134438634,
+      "tokens_seen": 2362114048
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.74065158080565e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.412337265554177,
+      "tokens_seen": 2362245120
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.739849141389826e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.4123221187402906,
+      "tokens_seen": 2362376192
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.739046701974001e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.4123069730020683,
+      "tokens_seen": 2362507264
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.738244262558177e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.412291828339374,
+      "tokens_seen": 2362638336
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.737441823142353e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4122766847520722,
+      "tokens_seen": 2362769408
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.736639383726529e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.412261542240026,
+      "tokens_seen": 2362900480
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.735836944310705e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.4122464008031,
+      "tokens_seen": 2363031552
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.73503450489488e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.4122312604411587,
+      "tokens_seen": 2363162624
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.734232065479057e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.412216121154065,
+      "tokens_seen": 2363293696
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.733429626063232e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.4122009829416835,
+      "tokens_seen": 2363424768
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.732627186647408e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.4121858458038785,
+      "tokens_seen": 2363555840
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.731824747231584e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.412170709740514,
+      "tokens_seen": 2363686912
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.73102230781576e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.4121555747514547,
+      "tokens_seen": 2363817984
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.730219868399936e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.4121404408365636,
+      "tokens_seen": 2363949056
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7294174289841115e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.412125307995706,
+      "tokens_seen": 2364080128
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.728614989568288e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.4121101762287456,
+      "tokens_seen": 2364211200
+    },
+    {
+      "epoch": 0.43,
+      "objective/train/docs_used": 793971,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2974660396575928,
+      "objective/train/theoretical_loss": 3.4120950455355468,
+      "objective/train/tokens_used": 734801376,
+      "theoretical_loss": 3.4120950455355468,
+      "tokens_seen": 2364342272
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7278125501524635e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.4120950455355468,
+      "tokens_seen": 2364342272
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7270101107366395e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.4120799159159736,
+      "tokens_seen": 2364473344
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7262076713208155e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.412064787369891,
+      "tokens_seen": 2364604416
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.725405231904991e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.412049659897163,
+      "tokens_seen": 2364735488
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7246027924891675e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.4120345334976534,
+      "tokens_seen": 2364866560
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.723800353073343e-05,
+      "loss": 1.1892,
+      "theoretical_loss": 3.412019408171227,
+      "tokens_seen": 2364997632
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7229979136575194e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.4120042839177485,
+      "tokens_seen": 2365128704
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.722195474241695e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.411989160737082,
+      "tokens_seen": 2365259776
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.721393034825871e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.411974038629092,
+      "tokens_seen": 2365390848
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.720590595410047e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.4119589175936436,
+      "tokens_seen": 2365521920
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.719788155994222e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.4119437976306,
+      "tokens_seen": 2365652992
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.718985716578399e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.411928678739827,
+      "tokens_seen": 2365784064
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.718183277162574e-05,
+      "loss": 1.3407,
+      "theoretical_loss": 3.4119135609211884,
+      "tokens_seen": 2365915136
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.717380837746751e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.411898444174549,
+      "tokens_seen": 2366046208
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.716578398330926e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.4118833284997736,
+      "tokens_seen": 2366177280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.715775958915101e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.4118682138967262,
+      "tokens_seen": 2366308352
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.714973519499278e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.4118531003652723,
+      "tokens_seen": 2366439424
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.714171080083453e-05,
+      "loss": 1.2962,
+      "theoretical_loss": 3.411837987905276,
+      "tokens_seen": 2366570496
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.71336864066763e-05,
+      "loss": 1.3013,
+      "theoretical_loss": 3.4118228765166023,
+      "tokens_seen": 2366701568
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.712566201251805e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.411807766199116,
+      "tokens_seen": 2366832640
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.711763761835982e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.4117926569526813,
+      "tokens_seen": 2366963712
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.710961322420157e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4117775487771635,
+      "tokens_seen": 2367094784
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.710158883004334e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.4117624416724275,
+      "tokens_seen": 2367225856
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.709356443588509e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.4117473356383377,
+      "tokens_seen": 2367356928
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.7085540041726846e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.411732230674759,
+      "tokens_seen": 2367488000
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.707751564756861e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.4117171267815563,
+      "tokens_seen": 2367619072
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.7069491253410365e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.411702023958595,
+      "tokens_seen": 2367750144
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.706146685925213e-05,
+      "loss": 1.2981,
+      "theoretical_loss": 3.4116869222057393,
+      "tokens_seen": 2367881216
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.7053442465093885e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4116718215228543,
+      "tokens_seen": 2368012288
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.704541807093565e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4116567219098055,
+      "tokens_seen": 2368143360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.7037393676777405e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.4116416233664575,
+      "tokens_seen": 2368274432
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.702936928261916e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.4116265258926752,
+      "tokens_seen": 2368405504
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.7021344888460925e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.411611429488324,
+      "tokens_seen": 2368536576
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.701332049430268e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.411596334153269,
+      "tokens_seen": 2368667648
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.7005296100144445e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.411581239887375,
+      "tokens_seen": 2368798720
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.69972717059862e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.411566146690507,
+      "tokens_seen": 2368929792
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6989247311827965e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.4115510545625303,
+      "tokens_seen": 2369060864
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.698122291766972e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.4115359635033102,
+      "tokens_seen": 2369191936
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.697319852351147e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.411520873512712,
+      "tokens_seen": 2369323008
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.696517412935324e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.4115057845906005,
+      "tokens_seen": 2369454080
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.695714973519499e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.4114906967368412,
+      "tokens_seen": 2369585152
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.694912534103676e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.4114756099512995,
+      "tokens_seen": 2369716224
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.694110094687851e-05,
+      "loss": 1.151,
+      "theoretical_loss": 3.4114605242338403,
+      "tokens_seen": 2369847296
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.693307655272028e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.4114454395843294,
+      "tokens_seen": 2369978368
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.692505215856203e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.411430356002632,
+      "tokens_seen": 2370109440
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.691702776440378e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.411415273488613,
+      "tokens_seen": 2370240512
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.690900337024555e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.4114001920421386,
+      "tokens_seen": 2370371584
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.69009789760873e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.4113851116630736,
+      "tokens_seen": 2370502656
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.689295458192907e-05,
+      "loss": 1.2921,
+      "theoretical_loss": 3.411370032351283,
+      "tokens_seen": 2370633728
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.688493018777082e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.411354954106634,
+      "tokens_seen": 2370764800
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 796080,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3346552848815918,
+      "objective/train/theoretical_loss": 3.4113398769289898,
+      "objective/train/tokens_used": 741354976,
+      "theoretical_loss": 3.4113398769289898,
+      "tokens_seen": 2370895872
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.687690579361259e-05,
+      "loss": 1.296,
+      "theoretical_loss": 3.4113398769289898,
+      "tokens_seen": 2370895872
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.686888139945434e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.411324800818218,
+      "tokens_seen": 2371026944
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6860857005296096e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.4113097257741827,
+      "tokens_seen": 2371158016
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.685283261113786e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.41129465179675,
+      "tokens_seen": 2371289088
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6844808216979616e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.411279578885786,
+      "tokens_seen": 2371420160
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.683678382282138e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.411264507041155,
+      "tokens_seen": 2371551232
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6828759428663136e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.4112494362627235,
+      "tokens_seen": 2371682304
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.68207350345049e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.4112343665503575,
+      "tokens_seen": 2371813376
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6812710640346655e-05,
+      "loss": 1.1936,
+      "theoretical_loss": 3.411219297903922,
+      "tokens_seen": 2371944448
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.680468624618841e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.411204230323283,
+      "tokens_seen": 2372075520
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6796661852030175e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.411189163808306,
+      "tokens_seen": 2372206592
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.678863745787193e-05,
+      "loss": 1.1681,
+      "theoretical_loss": 3.411174098358857,
+      "tokens_seen": 2372337664
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6780613063713695e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.4111590339748017,
+      "tokens_seen": 2372468736
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.677258866955545e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.411143970656006,
+      "tokens_seen": 2372599808
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6764564275397215e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.4111289084023357,
+      "tokens_seen": 2372730880
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.675653988123897e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.4111138472136564,
+      "tokens_seen": 2372861952
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6748515487080735e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.411098787089834,
+      "tokens_seen": 2372993024
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.674049109292249e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.411083728030735,
+      "tokens_seen": 2373124096
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.673246669876424e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.411068670036225,
+      "tokens_seen": 2373255168
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.672444230460601e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.4110536131061693,
+      "tokens_seen": 2373386240
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.671641791044776e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.4110385572404347,
+      "tokens_seen": 2373517312
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.670839351628953e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.411023502438887,
+      "tokens_seen": 2373648384
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.670036912213128e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.4110084487013923,
+      "tokens_seen": 2373779456
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.669234472797305e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.4109933960278163,
+      "tokens_seen": 2373910528
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.66843203338148e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.4109783444180253,
+      "tokens_seen": 2374041600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6676295939656553e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.4109632938718852,
+      "tokens_seen": 2374172672
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.666827154549832e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.4109482443892625,
+      "tokens_seen": 2374303744
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.666024715134007e-05,
+      "loss": 1.3316,
+      "theoretical_loss": 3.410933195970023,
+      "tokens_seen": 2374434816
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.665222275718184e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.4109181486140328,
+      "tokens_seen": 2374565888
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.664419836302359e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.4109031023211585,
+      "tokens_seen": 2374696960
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.663617396886536e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.4108880570912663,
+      "tokens_seen": 2374828032
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.662814957470711e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.4108730129242217,
+      "tokens_seen": 2374959104
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6620125180548866e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.410857969819892,
+      "tokens_seen": 2375090176
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.661210078639063e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.410842927778143,
+      "tokens_seen": 2375221248
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6604076392232386e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.410827886798841,
+      "tokens_seen": 2375352320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.659605199807415e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.4108128468818517,
+      "tokens_seen": 2375483392
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6588027603915906e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.4107978080270427,
+      "tokens_seen": 2375614464
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.658000320975767e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.4107827702342797,
+      "tokens_seen": 2375745536
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6571978815599425e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.4107677335034294,
+      "tokens_seen": 2375876608
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.656395442144118e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.410752697834358,
+      "tokens_seen": 2376007680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6555930027282945e-05,
+      "loss": 1.1896,
+      "theoretical_loss": 3.4107376632269313,
+      "tokens_seen": 2376138752
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.65479056331247e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4107226296810174,
+      "tokens_seen": 2376269824
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6539881238966465e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.410707597196481,
+      "tokens_seen": 2376400896
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.653185684480822e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.41069256577319,
+      "tokens_seen": 2376531968
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6523832450649985e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.410677535411011,
+      "tokens_seen": 2376663040
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.651580805649174e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.4106625061098095,
+      "tokens_seen": 2376794112
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.650778366233349e-05,
+      "loss": 1.2878,
+      "theoretical_loss": 3.4106474778694524,
+      "tokens_seen": 2376925184
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.649975926817526e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.410632450689807,
+      "tokens_seen": 2377056256
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.649173487401701e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.4106174245707397,
+      "tokens_seen": 2377187328
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.648371047985878e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.4106023995121166,
+      "tokens_seen": 2377318400
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 798224,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2504487037658691,
+      "objective/train/theoretical_loss": 3.410587375513805,
+      "objective/train/tokens_used": 747908576,
+      "theoretical_loss": 3.410587375513805,
+      "tokens_seen": 2377449472
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.647568608570053e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.410587375513805,
+      "tokens_seen": 2377449472
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.64676616915423e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.410572352575671,
+      "tokens_seen": 2377580544
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.645963729738405e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.4105573306975825,
+      "tokens_seen": 2377711616
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.645161290322582e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.410542309879405,
+      "tokens_seen": 2377842688
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.644358850906757e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.4105272901210064,
+      "tokens_seen": 2377973760
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6435564114909324e-05,
+      "loss": 1.1694,
+      "theoretical_loss": 3.4105122714222524,
+      "tokens_seen": 2378104832
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.642753972075109e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.410497253783011,
+      "tokens_seen": 2378235904
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.641951532659284e-05,
+      "loss": 1.2961,
+      "theoretical_loss": 3.410482237203148,
+      "tokens_seen": 2378366976
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.641149093243461e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.410467221682531,
+      "tokens_seen": 2378498048
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.640346653827636e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.4104522072210264,
+      "tokens_seen": 2378629120
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.639544214411813e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.4104371938185016,
+      "tokens_seen": 2378760192
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.638741774995988e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.4104221814748237,
+      "tokens_seen": 2378891264
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6379393355801636e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.4104071701898593,
+      "tokens_seen": 2379022336
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.63713689616434e-05,
+      "loss": 1.1914,
+      "theoretical_loss": 3.4103921599634752,
+      "tokens_seen": 2379153408
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6363344567485156e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.410377150795539,
+      "tokens_seen": 2379284480
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.635532017332692e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.4103621426859174,
+      "tokens_seen": 2379415552
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6347295779168676e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.4103471356344777,
+      "tokens_seen": 2379546624
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.633927138501044e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.4103321296410867,
+      "tokens_seen": 2379677696
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6331246990852196e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.410317124705612,
+      "tokens_seen": 2379808768
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.632322259669395e-05,
+      "loss": 1.2679,
+      "theoretical_loss": 3.4103021208279203,
+      "tokens_seen": 2379939840
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6315198202535715e-05,
+      "loss": 1.1782,
+      "theoretical_loss": 3.4102871180078793,
+      "tokens_seen": 2380070912
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.630717380837747e-05,
+      "loss": 1.2726,
+      "theoretical_loss": 3.4102721162453555,
+      "tokens_seen": 2380201984
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6299149414219235e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.4102571155402166,
+      "tokens_seen": 2380333056
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.629112502006099e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.4102421158923297,
+      "tokens_seen": 2380464128
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6283100625902755e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.410227117301562,
+      "tokens_seen": 2380595200
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.627507623174451e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.410212119767781,
+      "tokens_seen": 2380726272
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.626705183758626e-05,
+      "loss": 1.2986,
+      "theoretical_loss": 3.410197123290854,
+      "tokens_seen": 2380857344
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.625902744342803e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.4101821278706486,
+      "tokens_seen": 2380988416
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.625100304926978e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.4101671335070316,
+      "tokens_seen": 2381119488
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.624297865511155e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.410152140199871,
+      "tokens_seen": 2381250560
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.62349542609533e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.410137147949033,
+      "tokens_seen": 2381381632
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.622692986679506e-05,
+      "loss": 1.3107,
+      "theoretical_loss": 3.4101221567543867,
+      "tokens_seen": 2381512704
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.621890547263682e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.410107166615799,
+      "tokens_seen": 2381643776
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6210881078478574e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.4100921775331368,
+      "tokens_seen": 2381774848
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.620285668432034e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.410077189506268,
+      "tokens_seen": 2381905920
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6194832290162094e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.41006220253506,
+      "tokens_seen": 2382036992
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6186807896003853e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.410047216619381,
+      "tokens_seen": 2382168064
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6178783501845613e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.410032231759098,
+      "tokens_seen": 2382299136
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.617075910768737e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.4100172479540785,
+      "tokens_seen": 2382430208
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.616273471352913e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.4100022652041906,
+      "tokens_seen": 2382561280
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.615471031937089e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.4099872835093015,
+      "tokens_seen": 2382692352
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.614668592521265e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.409972302869279,
+      "tokens_seen": 2382823424
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6138661531054406e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.409957323283991,
+      "tokens_seen": 2382954496
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6130637136896166e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.4099423447533055,
+      "tokens_seen": 2383085568
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6122612742737926e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.40992736727709,
+      "tokens_seen": 2383216640
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6114588348579686e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.4099123908552116,
+      "tokens_seen": 2383347712
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6106563954421446e-05,
+      "loss": 1.1973,
+      "theoretical_loss": 3.4098974154875386,
+      "tokens_seen": 2383478784
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.6098539560263206e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.4098824411739392,
+      "tokens_seen": 2383609856
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.609051516610496e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.409867467914281,
+      "tokens_seen": 2383740928
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.608249077194672e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.409852495708432,
+      "tokens_seen": 2383872000
+    },
+    {
+      "epoch": 0.44,
+      "objective/train/docs_used": 800454,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2513808012008667,
+      "objective/train/theoretical_loss": 3.4098375245562593,
+      "objective/train/tokens_used": 754462176,
+      "theoretical_loss": 3.4098375245562593,
+      "tokens_seen": 2384003072
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.607446637778848e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.4098375245562593,
+      "tokens_seen": 2384003072
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.606644198363024e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.409822554457632,
+      "tokens_seen": 2384134144
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.6058417589472e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.4098075854124175,
+      "tokens_seen": 2384265216
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.605039319531376e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.4097926174204836,
+      "tokens_seen": 2384396288
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.604236880115552e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.409777650481699,
+      "tokens_seen": 2384527360
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.603434440699727e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.4097626845959303,
+      "tokens_seen": 2384658432
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.602632001283903e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.409747719763047,
+      "tokens_seen": 2384789504
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.601829561868079e-05,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4097327559829163,
+      "tokens_seen": 2384920576
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.601027122452255e-05,
+      "loss": 1.277,
+      "theoretical_loss": 3.4097177932554064,
+      "tokens_seen": 2385051648
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.600224683036431e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.4097028315803857,
+      "tokens_seen": 2385182720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5994222436206064e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.4096878709577223,
+      "tokens_seen": 2385313792
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.598619804204783e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.4096729113872843,
+      "tokens_seen": 2385444864
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5978173647889584e-05,
+      "loss": 1.1482,
+      "theoretical_loss": 3.40965795286894,
+      "tokens_seen": 2385575936
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5970149253731344e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4096429954025576,
+      "tokens_seen": 2385707008
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5962124859573104e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.409628038988005,
+      "tokens_seen": 2385838080
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5954100465414864e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4096130836251506,
+      "tokens_seen": 2385969152
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5946076071256624e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4095981293138626,
+      "tokens_seen": 2386100224
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.593805167709838e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.4095831760540096,
+      "tokens_seen": 2386231296
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.593002728294014e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4095682238454597,
+      "tokens_seen": 2386362368
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5922002888781897e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.4095532726880813,
+      "tokens_seen": 2386493440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5913978494623656e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.409538322581743,
+      "tokens_seen": 2386624512
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5905954100465416e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.4095233735263126,
+      "tokens_seen": 2386755584
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.589792970630717e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4095084255216594,
+      "tokens_seen": 2386886656
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5889905312148936e-05,
+      "loss": 1.312,
+      "theoretical_loss": 3.409493478567651,
+      "tokens_seen": 2387017728
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.588188091799069e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.4094785326641563,
+      "tokens_seen": 2387148800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5873856523832456e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.409463587811044,
+      "tokens_seen": 2387279872
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.586583212967421e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.409448644008182,
+      "tokens_seen": 2387410944
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.585780773551597e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4094337012554394,
+      "tokens_seen": 2387542016
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.584978334135773e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.4094187595526844,
+      "tokens_seen": 2387673088
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.584175894719948e-05,
+      "loss": 1.1631,
+      "theoretical_loss": 3.4094038188997855,
+      "tokens_seen": 2387804160
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.583373455304125e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.409388879296612,
+      "tokens_seen": 2387935232
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5825710158883e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.4093739407430315,
+      "tokens_seen": 2388066304
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.581768576472477e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.4093590032389134,
+      "tokens_seen": 2388197376
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.580966137056652e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.4093440667841266,
+      "tokens_seen": 2388328448
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.580163697640829e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.4093291313785388,
+      "tokens_seen": 2388459520
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.579361258225004e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4093141970220198,
+      "tokens_seen": 2388590592
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5785588188091795e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.4092992637144377,
+      "tokens_seen": 2388721664
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.577756379393356e-05,
+      "loss": 1.2709,
+      "theoretical_loss": 3.4092843314556616,
+      "tokens_seen": 2388852736
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5769539399775314e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.40926940024556,
+      "tokens_seen": 2388983808
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.576151500561708e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.4092544700840017,
+      "tokens_seen": 2389114880
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5753490611458834e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.4092395409708565,
+      "tokens_seen": 2389245952
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.57454662173006e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.4092246129059918,
+      "tokens_seen": 2389377024
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5737441823142354e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.4092096858892775,
+      "tokens_seen": 2389508096
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.572941742898411e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.409194759920582,
+      "tokens_seen": 2389639168
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5721393034825874e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.4091798349997746,
+      "tokens_seen": 2389770240
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.571336864066763e-05,
+      "loss": 1.2604,
+      "theoretical_loss": 3.409164911126724,
+      "tokens_seen": 2389901312
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5705344246509394e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.4091499883012997,
+      "tokens_seen": 2390032384
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.569731985235115e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.40913506652337,
+      "tokens_seen": 2390163456
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5689295458192913e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.409120145792804,
+      "tokens_seen": 2390294528
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5681271064034667e-05,
+      "loss": 1.3278,
+      "theoretical_loss": 3.4091052261094714,
+      "tokens_seen": 2390425600
+    },
+    {
+      "epoch": 0.45,
+      "objective/train/docs_used": 802547,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2286100387573242,
+      "objective/train/theoretical_loss": 3.4090903074732406,
+      "objective/train/tokens_used": 761015776,
+      "theoretical_loss": 3.4090903074732406,
+      "tokens_seen": 2390556672
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.567324666987642e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.4090903074732406,
+      "tokens_seen": 2390556672
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5665222275718186e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.409075389883981,
+      "tokens_seen": 2390687744
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.565719788155994e-05,
+      "loss": 1.2766,
+      "theoretical_loss": 3.409060473341562,
+      "tokens_seen": 2390818816
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5649173487401706e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.409045557845853,
+      "tokens_seen": 2390949888
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.564114909324346e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.4090306433967217,
+      "tokens_seen": 2391080960
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5633124699085226e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.4090157299940387,
+      "tokens_seen": 2391212032
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.562510030492698e-05,
+      "loss": 1.2746,
+      "theoretical_loss": 3.409000817637673,
+      "tokens_seen": 2391343104
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.561707591076873e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.4089859063274934,
+      "tokens_seen": 2391474176
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.56090515166105e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.408970996063369,
+      "tokens_seen": 2391605248
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.560102712245225e-05,
+      "loss": 1.3169,
+      "theoretical_loss": 3.40895608684517,
+      "tokens_seen": 2391736320
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.559300272829402e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.408941178672765,
+      "tokens_seen": 2391867392
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.558497833413577e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.408926271546024,
+      "tokens_seen": 2391998464
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.557695393997754e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.4089113654648155,
+      "tokens_seen": 2392129536
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.556892954581929e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.408896460429009,
+      "tokens_seen": 2392260608
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5560905151661045e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.408881556438475,
+      "tokens_seen": 2392391680
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.555288075750281e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.4088666534930816,
+      "tokens_seen": 2392522752
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5544856363344565e-05,
+      "loss": 1.3034,
+      "theoretical_loss": 3.408851751592699,
+      "tokens_seen": 2392653824
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.553683196918633e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.4088368507371967,
+      "tokens_seen": 2392784896
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5528807575028084e-05,
+      "loss": 1.3106,
+      "theoretical_loss": 3.4088219509264435,
+      "tokens_seen": 2392915968
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.552078318086985e-05,
+      "loss": 1.166,
+      "theoretical_loss": 3.4088070521603098,
+      "tokens_seen": 2393047040
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5512758786711604e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.408792154438665,
+      "tokens_seen": 2393178112
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.550473439255337e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.408777257761378,
+      "tokens_seen": 2393309184
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5496709998395124e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.408762362128319,
+      "tokens_seen": 2393440256
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.548868560423688e-05,
+      "loss": 1.2985,
+      "theoretical_loss": 3.4087474675393574,
+      "tokens_seen": 2393571328
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5480661210078644e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.4087325739943632,
+      "tokens_seen": 2393702400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.54726368159204e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.4087176814932056,
+      "tokens_seen": 2393833472
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5464612421762164e-05,
+      "loss": 1.3033,
+      "theoretical_loss": 3.4087027900357545,
+      "tokens_seen": 2393964544
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.545658802760392e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.4086878996218797,
+      "tokens_seen": 2394095616
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5448563633445684e-05,
+      "loss": 1.3372,
+      "theoretical_loss": 3.408673010251451,
+      "tokens_seen": 2394226688
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.544053923928744e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.4086581219243377,
+      "tokens_seen": 2394357760
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.543251484512919e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.40864323464041,
+      "tokens_seen": 2394488832
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5424490450970956e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.408628348399538,
+      "tokens_seen": 2394619904
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.541646605681271e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4086134632015903,
+      "tokens_seen": 2394750976
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5408441662654476e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.4085985790464384,
+      "tokens_seen": 2394882048
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.540041726849623e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.4085836959339506,
+      "tokens_seen": 2395013120
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5392392874337996e-05,
+      "loss": 1.1687,
+      "theoretical_loss": 3.4085688138639982,
+      "tokens_seen": 2395144192
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.538436848017975e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.4085539328364503,
+      "tokens_seen": 2395275264
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.53763440860215e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.408539052851177,
+      "tokens_seen": 2395406336
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.536831969186327e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.4085241739080487,
+      "tokens_seen": 2395537408
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.536029529770502e-05,
+      "loss": 1.3283,
+      "theoretical_loss": 3.4085092960069345,
+      "tokens_seen": 2395668480
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.535227090354679e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.408494419147705,
+      "tokens_seen": 2395799552
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.534424650938854e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.4084795433302304,
+      "tokens_seen": 2395930624
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.533622211523031e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.4084646685543807,
+      "tokens_seen": 2396061696
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.532819772107206e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.408449794820026,
+      "tokens_seen": 2396192768
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5320173326913815e-05,
+      "loss": 1.3619,
+      "theoretical_loss": 3.408434922127036,
+      "tokens_seen": 2396323840
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.531214893275558e-05,
+      "loss": 1.2785,
+      "theoretical_loss": 3.408420050475281,
+      "tokens_seen": 2396454912
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5304124538597335e-05,
+      "loss": 1.3445,
+      "theoretical_loss": 3.408405179864631,
+      "tokens_seen": 2396585984
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.52961001444391e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4083903102949566,
+      "tokens_seen": 2396717056
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5288075750280855e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.408375441766128,
+      "tokens_seen": 2396848128
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.528005135612262e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.4083605742780154,
+      "tokens_seen": 2396979200
+    },
+    {
+      "epoch": 0.45,
+      "objective/train/docs_used": 804843,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2564585208892822,
+      "objective/train/theoretical_loss": 3.4083457078304886,
+      "objective/train/tokens_used": 767569376,
+      "theoretical_loss": 3.4083457078304886,
+      "tokens_seen": 2397110272
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5272026961964374e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.4083457078304886,
+      "tokens_seen": 2397110272
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.526400256780613e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.4083308424234184,
+      "tokens_seen": 2397241344
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5255978173647894e-05,
+      "loss": 1.2693,
+      "theoretical_loss": 3.4083159780566747,
+      "tokens_seen": 2397372416
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.524795377948965e-05,
+      "loss": 1.3269,
+      "theoretical_loss": 3.4083011147301283,
+      "tokens_seen": 2397503488
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5239929385331414e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.4082862524436495,
+      "tokens_seen": 2397634560
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.523190499117317e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.4082713911971076,
+      "tokens_seen": 2397765632
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5223880597014934e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.408256530990375,
+      "tokens_seen": 2397896704
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.521585620285669e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.40824167182332,
+      "tokens_seen": 2398027776
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5207831808698454e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.4082268136958143,
+      "tokens_seen": 2398158848
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.519980741454021e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.408211956607728,
+      "tokens_seen": 2398289920
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.519178302038196e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.4081971005589318,
+      "tokens_seen": 2398420992
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5183758626223727e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.4081822455492956,
+      "tokens_seen": 2398552064
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.517573423206548e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.408167391578691,
+      "tokens_seen": 2398683136
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5167709837907246e-05,
+      "loss": 1.3109,
+      "theoretical_loss": 3.4081525386469878,
+      "tokens_seen": 2398814208
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5159685443749e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4081376867540563,
+      "tokens_seen": 2398945280
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5151661049590766e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.4081228358997677,
+      "tokens_seen": 2399076352
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.514363665543252e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.4081079860839925,
+      "tokens_seen": 2399207424
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.513561226127427e-05,
+      "loss": 1.3295,
+      "theoretical_loss": 3.4080931373066012,
+      "tokens_seen": 2399338496
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.512758786711604e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.408078289567465,
+      "tokens_seen": 2399469568
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.511956347295779e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.4080634428664536,
+      "tokens_seen": 2399600640
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.511153907879956e-05,
+      "loss": 1.2877,
+      "theoretical_loss": 3.4080485972034387,
+      "tokens_seen": 2399731712
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.510351468464131e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.40803375257829,
+      "tokens_seen": 2399862784
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.509549029048308e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.4080189089908792,
+      "tokens_seen": 2399993856
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.508746589632483e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.408004066441077,
+      "tokens_seen": 2400124928
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5079441502166585e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.4079892249287536,
+      "tokens_seen": 2400256000
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.507141710800835e-05,
+      "loss": 1.2293,
+      "theoretical_loss": 3.40797438445378,
+      "tokens_seen": 2400387072
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.5063392713850105e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.407959545016028,
+      "tokens_seen": 2400518144
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 5.505536831969187e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.407944706615367,
+      "tokens_seen": 2400649216
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.5047343925533625e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.4079298692516695,
+      "tokens_seen": 2400780288
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.503931953137539e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.407915032924805,
+      "tokens_seen": 2400911360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.5031295137217144e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.407900197634645,
+      "tokens_seen": 2401042432
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.50232707430589e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.4078853633810606,
+      "tokens_seen": 2401173504
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.5015246348900664e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.4078705301639225,
+      "tokens_seen": 2401304576
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.500722195474242e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.407855697983102,
+      "tokens_seen": 2401435648
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4999197560584184e-05,
+      "loss": 1.3292,
+      "theoretical_loss": 3.40784086683847,
+      "tokens_seen": 2401566720
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.499117316642594e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.407826036729898,
+      "tokens_seen": 2401697792
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4983148772267704e-05,
+      "loss": 1.3634,
+      "theoretical_loss": 3.407811207657256,
+      "tokens_seen": 2401828864
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.497512437810946e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.4077963796204163,
+      "tokens_seen": 2401959936
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.496709998395121e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.4077815526192494,
+      "tokens_seen": 2402091008
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.495907558979298e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.4077667266536267,
+      "tokens_seen": 2402222080
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.495105119563473e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.407751901723419,
+      "tokens_seen": 2402353152
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.49430268014765e-05,
+      "loss": 1.3022,
+      "theoretical_loss": 3.407737077828498,
+      "tokens_seen": 2402484224
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.493500240731825e-05,
+      "loss": 1.2903,
+      "theoretical_loss": 3.4077222549687343,
+      "tokens_seen": 2402615296
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4926978013160016e-05,
+      "loss": 1.2746,
+      "theoretical_loss": 3.407707433144,
+      "tokens_seen": 2402746368
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.491895361900177e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.4076926123541655,
+      "tokens_seen": 2402877440
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.491092922484352e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.407677792599103,
+      "tokens_seen": 2403008512
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.490290483068529e-05,
+      "loss": 1.1887,
+      "theoretical_loss": 3.4076629738786828,
+      "tokens_seen": 2403139584
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.489488043652704e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.4076481561927774,
+      "tokens_seen": 2403270656
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.488685604236881e-05,
+      "loss": 1.1856,
+      "theoretical_loss": 3.407633339541257,
+      "tokens_seen": 2403401728
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.487883164821056e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.407618523923994,
+      "tokens_seen": 2403532800
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 806890,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2402286529541016,
+      "objective/train/theoretical_loss": 3.407603709340859,
+      "objective/train/tokens_used": 774122976,
+      "theoretical_loss": 3.407603709340859,
+      "tokens_seen": 2403663872
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.487080725405232e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.407603709340859,
+      "tokens_seen": 2403663872
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.486278285989408e-05,
+      "loss": 1.1846,
+      "theoretical_loss": 3.407588895791724,
+      "tokens_seen": 2403794944
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.485475846573584e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.40757408327646,
+      "tokens_seen": 2403926016
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.48467340715776e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.4075592717949386,
+      "tokens_seen": 2404057088
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4838709677419355e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.4075444613470314,
+      "tokens_seen": 2404188160
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.483068528326112e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.4075296519326104,
+      "tokens_seen": 2404319232
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4822660889102875e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.4075148435515463,
+      "tokens_seen": 2404450304
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4814636494944635e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.407500036203711,
+      "tokens_seen": 2404581376
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4806612100786395e-05,
+      "loss": 1.1938,
+      "theoretical_loss": 3.407485229888976,
+      "tokens_seen": 2404712448
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4798587706628155e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4074704246072134,
+      "tokens_seen": 2404843520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4790563312469914e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.407455620358294,
+      "tokens_seen": 2404974592
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.478253891831167e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.4074408171420907,
+      "tokens_seen": 2405105664
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.477451452415343e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.407426014958474,
+      "tokens_seen": 2405236736
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.476649012999519e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.407411213807316,
+      "tokens_seen": 2405367808
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.475846573583695e-05,
+      "loss": 1.1935,
+      "theoretical_loss": 3.4073964136884887,
+      "tokens_seen": 2405498880
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.475044134167871e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4073816146018636,
+      "tokens_seen": 2405629952
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.474241694752047e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.407366816547312,
+      "tokens_seen": 2405761024
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.473439255336223e-05,
+      "loss": 1.1425,
+      "theoretical_loss": 3.407352019524707,
+      "tokens_seen": 2405892096
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.472636815920398e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.4073372235339194,
+      "tokens_seen": 2406023168
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.471834376504574e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.407322428574821,
+      "tokens_seen": 2406154240
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.47103193708875e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.4073076346472844,
+      "tokens_seen": 2406285312
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.470229497672926e-05,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4072928417511807,
+      "tokens_seen": 2406416384
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.469427058257102e-05,
+      "loss": 1.1712,
+      "theoretical_loss": 3.407278049886382,
+      "tokens_seen": 2406547456
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.468624618841278e-05,
+      "loss": 1.2981,
+      "theoretical_loss": 3.4072632590527605,
+      "tokens_seen": 2406678528
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.467822179425453e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.407248469250188,
+      "tokens_seen": 2406809600
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.467019740009629e-05,
+      "loss": 1.1843,
+      "theoretical_loss": 3.4072336804785373,
+      "tokens_seen": 2406940672
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.466217300593805e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4072188927376788,
+      "tokens_seen": 2407071744
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.465414861177981e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4072041060274856,
+      "tokens_seen": 2407202816
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.464612421762157e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.4071893203478294,
+      "tokens_seen": 2407333888
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.463809982346333e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.4071745356985823,
+      "tokens_seen": 2407464960
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.463007542930509e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.407159752079617,
+      "tokens_seen": 2407596032
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4622051035146845e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.4071449694908047,
+      "tokens_seen": 2407727104
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4614026640988605e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.407130187932018,
+      "tokens_seen": 2407858176
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4606002246830365e-05,
+      "loss": 1.1771,
+      "theoretical_loss": 3.407115407403129,
+      "tokens_seen": 2407989248
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4597977852672125e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.4071006279040104,
+      "tokens_seen": 2408120320
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4589953458513885e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.4070858494345333,
+      "tokens_seen": 2408251392
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.458192906435564e-05,
+      "loss": 1.3388,
+      "theoretical_loss": 3.407071071994571,
+      "tokens_seen": 2408382464
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4573904670197405e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.407056295583995,
+      "tokens_seen": 2408513536
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.456588027603916e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.407041520202678,
+      "tokens_seen": 2408644608
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4557855881880925e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.407026745850492,
+      "tokens_seen": 2408775680
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.454983148772268e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.40701197252731,
+      "tokens_seen": 2408906752
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.454180709356443e-05,
+      "loss": 1.2775,
+      "theoretical_loss": 3.406997200233003,
+      "tokens_seen": 2409037824
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.45337826994062e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.4069824289674453,
+      "tokens_seen": 2409168896
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.452575830524795e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.4069676587305078,
+      "tokens_seen": 2409299968
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.451773391108972e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.406952889522063,
+      "tokens_seen": 2409431040
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.450970951693147e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.4069381213419843,
+      "tokens_seen": 2409562112
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.450168512277324e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.4069233541901434,
+      "tokens_seen": 2409693184
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.449366072861499e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.4069085880664125,
+      "tokens_seen": 2409824256
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4485636334456743e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.406893822970665,
+      "tokens_seen": 2409955328
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.447761194029851e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.406879058902773,
+      "tokens_seen": 2410086400
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 808982,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1448551416397095,
+      "objective/train/theoretical_loss": 3.4068642958626087,
+      "objective/train/tokens_used": 780676576,
+      "theoretical_loss": 3.4068642958626087,
+      "tokens_seen": 2410217472
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.446958754614026e-05,
+      "loss": 1.1517,
+      "theoretical_loss": 3.4068642958626087,
+      "tokens_seen": 2410217472
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.446156315198203e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.4068495338500453,
+      "tokens_seen": 2410348544
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.445353875782378e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.406834772864955,
+      "tokens_seen": 2410479616
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.444551436366555e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.40682001290721,
+      "tokens_seen": 2410610688
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.44374899695073e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4068052539766844,
+      "tokens_seen": 2410741760
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4429465575349056e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.406790496073249,
+      "tokens_seen": 2410872832
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.442144118119082e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.406775739196778,
+      "tokens_seen": 2411003904
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4413416787032576e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.406760983347144,
+      "tokens_seen": 2411134976
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.440539239287434e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4067462285242187,
+      "tokens_seen": 2411266048
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4397367998716096e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.4067314747278754,
+      "tokens_seen": 2411397120
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.438934360455786e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.406716721957987,
+      "tokens_seen": 2411528192
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4381319210399615e-05,
+      "loss": 1.3299,
+      "theoretical_loss": 3.406701970214426,
+      "tokens_seen": 2411659264
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.437329481624137e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.4066872194970657,
+      "tokens_seen": 2411790336
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4365270422083135e-05,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4066724698057786,
+      "tokens_seen": 2411921408
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.435724602792489e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4066577211404376,
+      "tokens_seen": 2412052480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4349221633766655e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.406642973500916,
+      "tokens_seen": 2412183552
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.434119723960841e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.406628226887086,
+      "tokens_seen": 2412314624
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4333172845450175e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.406613481298821,
+      "tokens_seen": 2412445696
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.432514845129193e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.406598736735994,
+      "tokens_seen": 2412576768
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.431712405713368e-05,
+      "loss": 1.2927,
+      "theoretical_loss": 3.4065839931984776,
+      "tokens_seen": 2412707840
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.430909966297545e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.4065692506861454,
+      "tokens_seen": 2412838912
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.43010752688172e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4065545091988696,
+      "tokens_seen": 2412969984
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.429305087465897e-05,
+      "loss": 1.321,
+      "theoretical_loss": 3.406539768736524,
+      "tokens_seen": 2413101056
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.428502648050072e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.4065250292989817,
+      "tokens_seen": 2413232128
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.427700208634249e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.406510290886115,
+      "tokens_seen": 2413363200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.426897769218424e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.4064955534977983,
+      "tokens_seen": 2413494272
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.426095329802601e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.4064808171339034,
+      "tokens_seen": 2413625344
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.425292890386776e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4064660817943038,
+      "tokens_seen": 2413756416
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4244904509709514e-05,
+      "loss": 1.2611,
+      "theoretical_loss": 3.4064513474788733,
+      "tokens_seen": 2413887488
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.423688011555128e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.4064366141874847,
+      "tokens_seen": 2414018560
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.422885572139303e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.4064218819200116,
+      "tokens_seen": 2414149632
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.42208313272348e-05,
+      "loss": 1.3354,
+      "theoretical_loss": 3.4064071506763263,
+      "tokens_seen": 2414280704
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.421280693307655e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.406392420456303,
+      "tokens_seen": 2414411776
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.420478253891832e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.4063776912598147,
+      "tokens_seen": 2414542848
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.419675814476007e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.406362963086735,
+      "tokens_seen": 2414673920
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4188733750601826e-05,
+      "loss": 1.3357,
+      "theoretical_loss": 3.4063482359369366,
+      "tokens_seen": 2414804992
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.418070935644359e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.406333509810293,
+      "tokens_seen": 2414936064
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4172684962285346e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.4063187847066785,
+      "tokens_seen": 2415067136
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.416466056812711e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.4063040606259656,
+      "tokens_seen": 2415198208
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4156636173968866e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.406289337568028,
+      "tokens_seen": 2415329280
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.414861177981063e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.406274615532739,
+      "tokens_seen": 2415460352
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4140587385652386e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.4062598945199722,
+      "tokens_seen": 2415591424
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.413256299149414e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.406245174529601,
+      "tokens_seen": 2415722496
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4124538597335905e-05,
+      "loss": 1.1834,
+      "theoretical_loss": 3.4062304555614995,
+      "tokens_seen": 2415853568
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.411651420317766e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.4062157376155406,
+      "tokens_seen": 2415984640
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4108489809019425e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.406201020691598,
+      "tokens_seen": 2416115712
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.410046541486118e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.406186304789545,
+      "tokens_seen": 2416246784
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.4092441020702945e-05,
+      "loss": 1.3213,
+      "theoretical_loss": 3.406171589909256,
+      "tokens_seen": 2416377856
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.40844166265447e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.4061568760506042,
+      "tokens_seen": 2416508928
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.407639223238645e-05,
+      "loss": 1.2522,
+      "theoretical_loss": 3.406142163213463,
+      "tokens_seen": 2416640000
+    },
+    {
+      "epoch": 0.46,
+      "objective/train/docs_used": 811333,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2345964908599854,
+      "objective/train/theoretical_loss": 3.406127451397707,
+      "objective/train/tokens_used": 787230176,
+      "theoretical_loss": 3.406127451397707,
+      "tokens_seen": 2416771072
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.406836783822822e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.406127451397707,
+      "tokens_seen": 2416771072
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.406034344406997e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.406112740603209,
+      "tokens_seen": 2416902144
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.405231904991174e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.4060980308298427,
+      "tokens_seen": 2417033216
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 5.404429465575349e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.406083322077482,
+      "tokens_seen": 2417164288
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.403627026159526e-05,
+      "loss": 1.2679,
+      "theoretical_loss": 3.4060686143460015,
+      "tokens_seen": 2417295360
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.402824586743701e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.406053907635274,
+      "tokens_seen": 2417426432
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.4020221473278764e-05,
+      "loss": 1.1909,
+      "theoretical_loss": 3.4060392019451737,
+      "tokens_seen": 2417557504
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.401219707912053e-05,
+      "loss": 1.3332,
+      "theoretical_loss": 3.406024497275574,
+      "tokens_seen": 2417688576
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.4004172684962284e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.4060097936263496,
+      "tokens_seen": 2417819648
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.399614829080405e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.4059950909973735,
+      "tokens_seen": 2417950720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3988123896645803e-05,
+      "loss": 1.2522,
+      "theoretical_loss": 3.4059803893885205,
+      "tokens_seen": 2418081792
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.398009950248757e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.4059656887996637,
+      "tokens_seen": 2418212864
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.397207510832932e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.4059509892306776,
+      "tokens_seen": 2418343936
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3964050714171076e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.405936290681436,
+      "tokens_seen": 2418475008
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.395602632001284e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.4059215931518128,
+      "tokens_seen": 2418606080
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3948001925854596e-05,
+      "loss": 1.3294,
+      "theoretical_loss": 3.405906896641682,
+      "tokens_seen": 2418737152
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.393997753169636e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.405892201150918,
+      "tokens_seen": 2418868224
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3931953137538116e-05,
+      "loss": 1.1869,
+      "theoretical_loss": 3.4058775066793943,
+      "tokens_seen": 2418999296
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.392392874337988e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.4058628132269857,
+      "tokens_seen": 2419130368
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3915904349221636e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.4058481207935656,
+      "tokens_seen": 2419261440
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.39078799550634e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.4058334293790082,
+      "tokens_seen": 2419392512
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3899855560905156e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.4058187389831884,
+      "tokens_seen": 2419523584
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.389183116674691e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4058040496059796,
+      "tokens_seen": 2419654656
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3883806772588675e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.4057893612472565,
+      "tokens_seen": 2419785728
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.387578237843043e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.405774673906893,
+      "tokens_seen": 2419916800
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3867757984272195e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.405759987584763,
+      "tokens_seen": 2420047872
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.385973359011395e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.405745302280742,
+      "tokens_seen": 2420178944
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3851709195955715e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.405730617994703,
+      "tokens_seen": 2420310016
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.384368480179747e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.405715934726521,
+      "tokens_seen": 2420441088
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.383566040763922e-05,
+      "loss": 1.1576,
+      "theoretical_loss": 3.40570125247607,
+      "tokens_seen": 2420572160
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.382763601348099e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.4056865712432245,
+      "tokens_seen": 2420703232
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.381961161932274e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.4056718910278585,
+      "tokens_seen": 2420834304
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.381158722516451e-05,
+      "loss": 1.2273,
+      "theoretical_loss": 3.405657211829847,
+      "tokens_seen": 2420965376
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.380356283100626e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.4056425336490648,
+      "tokens_seen": 2421096448
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.379553843684803e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.405627856485385,
+      "tokens_seen": 2421227520
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.378751404268978e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.405613180338683,
+      "tokens_seen": 2421358592
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3779489648531534e-05,
+      "loss": 1.2314,
+      "theoretical_loss": 3.405598505208833,
+      "tokens_seen": 2421489664
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.37714652543733e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.4055838310957096,
+      "tokens_seen": 2421620736
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3763440860215054e-05,
+      "loss": 1.2938,
+      "theoretical_loss": 3.405569157999188,
+      "tokens_seen": 2421751808
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.375541646605682e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.405554485919141,
+      "tokens_seen": 2421882880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3747392071898574e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.405539814855445,
+      "tokens_seen": 2422013952
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.373936767774034e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.4055251448079735,
+      "tokens_seen": 2422145024
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.373134328358209e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4055104757766017,
+      "tokens_seen": 2422276096
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3723318889423846e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.405495807761204,
+      "tokens_seen": 2422407168
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.371529449526561e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.4054811407616548,
+      "tokens_seen": 2422538240
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3707270101107366e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.4054664747778296,
+      "tokens_seen": 2422669312
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.369924570694913e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.4054518098096023,
+      "tokens_seen": 2422800384
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3691221312790886e-05,
+      "loss": 1.2611,
+      "theoretical_loss": 3.405437145856848,
+      "tokens_seen": 2422931456
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.368319691863265e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.4054224829194415,
+      "tokens_seen": 2423062528
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3675172524474406e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.405407820997257,
+      "tokens_seen": 2423193600
+    },
+    {
+      "epoch": 0.47,
+      "objective/train/docs_used": 813659,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2890591621398926,
+      "objective/train/theoretical_loss": 3.40539316009017,
+      "objective/train/tokens_used": 793783776,
+      "theoretical_loss": 3.40539316009017,
+      "tokens_seen": 2423324672
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.366714813031616e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.40539316009017,
+      "tokens_seen": 2423324672
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3659123736157926e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.405378500198055,
+      "tokens_seen": 2423455744
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.365109934199968e-05,
+      "loss": 1.222,
+      "theoretical_loss": 3.4053638413207876,
+      "tokens_seen": 2423586816
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3643074947841446e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.4053491834582417,
+      "tokens_seen": 2423717888
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.36350505536832e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.405334526610292,
+      "tokens_seen": 2423848960
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3627026159524965e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.4053198707768146,
+      "tokens_seen": 2423980032
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.361900176536672e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.405305215957683,
+      "tokens_seen": 2424111104
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.361097737120848e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.4052905621527736,
+      "tokens_seen": 2424242176
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.360295297705024e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.40527590936196,
+      "tokens_seen": 2424373248
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.359492858289199e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.4052612575851184,
+      "tokens_seen": 2424504320
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.358690418873376e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.405246606822123,
+      "tokens_seen": 2424635392
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.357887979457551e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.405231957072849,
+      "tokens_seen": 2424766464
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.357085540041728e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.405217308337172,
+      "tokens_seen": 2424897536
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.356283100625903e-05,
+      "loss": 1.2859,
+      "theoretical_loss": 3.405202660614967,
+      "tokens_seen": 2425028608
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.355480661210079e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.405188013906108,
+      "tokens_seen": 2425159680
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.354678221794255e-05,
+      "loss": 1.1997,
+      "theoretical_loss": 3.4051733682104715,
+      "tokens_seen": 2425290752
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3538757823784304e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.4051587235279315,
+      "tokens_seen": 2425421824
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.353073342962607e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.4051440798583643,
+      "tokens_seen": 2425552896
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3522709035467824e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4051294372016443,
+      "tokens_seen": 2425683968
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3514684641309584e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.405114795557647,
+      "tokens_seen": 2425815040
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3506660247151344e-05,
+      "loss": 1.1954,
+      "theoretical_loss": 3.405100154926248,
+      "tokens_seen": 2425946112
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3498635852993104e-05,
+      "loss": 1.2169,
+      "theoretical_loss": 3.405085515307322,
+      "tokens_seen": 2426077184
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3490611458834863e-05,
+      "loss": 1.3169,
+      "theoretical_loss": 3.4050708767007443,
+      "tokens_seen": 2426208256
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3482587064676617e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.4050562391063903,
+      "tokens_seen": 2426339328
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.347456267051838e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.4050416025241357,
+      "tokens_seen": 2426470400
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3466538276360136e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.4050269669538555,
+      "tokens_seen": 2426601472
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3458513882201896e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.405012332395425,
+      "tokens_seen": 2426732544
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3450489488043656e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.40499769884872,
+      "tokens_seen": 2426863616
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3442465093885416e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4049830663136156,
+      "tokens_seen": 2426994688
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3434440699727176e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.4049684347899873,
+      "tokens_seen": 2427125760
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.342641630556893e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.404953804277711,
+      "tokens_seen": 2427256832
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.341839191141069e-05,
+      "loss": 1.3123,
+      "theoretical_loss": 3.4049391747766613,
+      "tokens_seen": 2427387904
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.341036751725245e-05,
+      "loss": 1.1702,
+      "theoretical_loss": 3.4049245462867144,
+      "tokens_seen": 2427518976
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.340234312309421e-05,
+      "loss": 1.3444,
+      "theoretical_loss": 3.4049099188077454,
+      "tokens_seen": 2427650048
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.339431872893597e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.40489529233963,
+      "tokens_seen": 2427781120
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.338629433477773e-05,
+      "loss": 1.3465,
+      "theoretical_loss": 3.404880666882244,
+      "tokens_seen": 2427912192
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.337826994061949e-05,
+      "loss": 1.2709,
+      "theoretical_loss": 3.404866042435463,
+      "tokens_seen": 2428043264
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.337024554646124e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.4048514189991623,
+      "tokens_seen": 2428174336
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3362221152303e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.4048367965732176,
+      "tokens_seen": 2428305408
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.335419675814476e-05,
+      "loss": 1.3038,
+      "theoretical_loss": 3.4048221751575047,
+      "tokens_seen": 2428436480
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.334617236398652e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.4048075547518994,
+      "tokens_seen": 2428567552
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.333814796982828e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.404792935356277,
+      "tokens_seen": 2428698624
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.333012357567004e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.4047783169705137,
+      "tokens_seen": 2428829696
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3322099181511794e-05,
+      "loss": 1.2693,
+      "theoretical_loss": 3.404763699594485,
+      "tokens_seen": 2428960768
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.331407478735356e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.4047490832280665,
+      "tokens_seen": 2429091840
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3306050393195314e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.404734467871134,
+      "tokens_seen": 2429222912
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3298025999037074e-05,
+      "loss": 1.2817,
+      "theoretical_loss": 3.4047198535235643,
+      "tokens_seen": 2429353984
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3290001604878834e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.4047052401852316,
+      "tokens_seen": 2429485056
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3281977210720594e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.4046906278560134,
+      "tokens_seen": 2429616128
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3273952816562354e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.4046760165357846,
+      "tokens_seen": 2429747200
+    },
+    {
+      "epoch": 0.47,
+      "objective/train/docs_used": 815865,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2329602241516113,
+      "objective/train/theoretical_loss": 3.4046614062244207,
+      "objective/train/tokens_used": 800337376,
+      "theoretical_loss": 3.4046614062244207,
+      "tokens_seen": 2429878272
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.326592842240411e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.4046614062244207,
+      "tokens_seen": 2429878272
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3257904028245874e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.404646796921799,
+      "tokens_seen": 2430009344
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.324987963408763e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.404632188627794,
+      "tokens_seen": 2430140416
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.324185523992939e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.4046175813422828,
+      "tokens_seen": 2430271488
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3233830845771147e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.4046029750651408,
+      "tokens_seen": 2430402560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.32258064516129e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.404588369796244,
+      "tokens_seen": 2430533632
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3217782057454666e-05,
+      "loss": 1.2105,
+      "theoretical_loss": 3.404573765535469,
+      "tokens_seen": 2430664704
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.320975766329642e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.404559162282691,
+      "tokens_seen": 2430795776
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3201733269138186e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.404544560037787,
+      "tokens_seen": 2430926848
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.319370887497994e-05,
+      "loss": 1.2908,
+      "theoretical_loss": 3.404529958800632,
+      "tokens_seen": 2431057920
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.31856844808217e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.4045153585711034,
+      "tokens_seen": 2431188992
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.317766008666346e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.404500759349076,
+      "tokens_seen": 2431320064
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.316963569250521e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.404486161134427,
+      "tokens_seen": 2431451136
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.316161129834698e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4044715639270327,
+      "tokens_seen": 2431582208
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.315358690418873e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.4044569677267686,
+      "tokens_seen": 2431713280
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.31455625100305e-05,
+      "loss": 1.2815,
+      "theoretical_loss": 3.404442372533511,
+      "tokens_seen": 2431844352
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.313753811587225e-05,
+      "loss": 1.185,
+      "theoretical_loss": 3.404427778347136,
+      "tokens_seen": 2431975424
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3129513721714005e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.4044131851675212,
+      "tokens_seen": 2432106496
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.312148932755577e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4043985929945415,
+      "tokens_seen": 2432237568
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3113464933397525e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.4043840018280735,
+      "tokens_seen": 2432368640
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.310544053923929e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.404369411667994,
+      "tokens_seen": 2432499712
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3097416145081045e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4043548225141786,
+      "tokens_seen": 2432630784
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.308939175092281e-05,
+      "loss": 1.3037,
+      "theoretical_loss": 3.4043402343665043,
+      "tokens_seen": 2432761856
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3081367356764564e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.4043256472248475,
+      "tokens_seen": 2432892928
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.307334296260632e-05,
+      "loss": 1.2859,
+      "theoretical_loss": 3.4043110610890848,
+      "tokens_seen": 2433024000
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3065318568448084e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.4042964759590917,
+      "tokens_seen": 2433155072
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.305729417428984e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.4042818918347457,
+      "tokens_seen": 2433286144
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3049269780131604e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.404267308715923,
+      "tokens_seen": 2433417216
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.304124538597336e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.4042527266024996,
+      "tokens_seen": 2433548288
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 5.3033220991815124e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.404238145494353,
+      "tokens_seen": 2433679360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.302519659765688e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.4042235653913586,
+      "tokens_seen": 2433810432
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.301717220349863e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.404208986293394,
+      "tokens_seen": 2433941504
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.30091478093404e-05,
+      "loss": 1.1714,
+      "theoretical_loss": 3.4041944082003353,
+      "tokens_seen": 2434072576
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.300112341518215e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.4041798311120592,
+      "tokens_seen": 2434203648
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2993099021023917e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.4041652550284427,
+      "tokens_seen": 2434334720
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.298507462686567e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.404150679949362,
+      "tokens_seen": 2434465792
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2977050232707436e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.404136105874694,
+      "tokens_seen": 2434596864
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.296902583854919e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.4041215328043153,
+      "tokens_seen": 2434727936
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2961001444390956e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.4041069607381025,
+      "tokens_seen": 2434859008
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.295297705023271e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.4040923896759328,
+      "tokens_seen": 2434990080
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.294495265607446e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.4040778196176826,
+      "tokens_seen": 2435121152
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.293692826191623e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.404063250563229,
+      "tokens_seen": 2435252224
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.292890386775798e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.4040486825124487,
+      "tokens_seen": 2435383296
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.292087947359975e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.404034115465218,
+      "tokens_seen": 2435514368
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.29128550794415e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.404019549421415,
+      "tokens_seen": 2435645440
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.290483068528327e-05,
+      "loss": 1.2916,
+      "theoretical_loss": 3.4040049843809155,
+      "tokens_seen": 2435776512
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.289680629112502e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.4039904203435967,
+      "tokens_seen": 2435907584
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2888781896966775e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.4039758573093355,
+      "tokens_seen": 2436038656
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.288075750280854e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.4039612952780094,
+      "tokens_seen": 2436169728
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2872733108650295e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.4039467342494945,
+      "tokens_seen": 2436300800
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 817860,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2601072788238525,
+      "objective/train/theoretical_loss": 3.4039321742236686,
+      "objective/train/tokens_used": 806890976,
+      "theoretical_loss": 3.4039321742236686,
+      "tokens_seen": 2436431872
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.286470871449206e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.4039321742236686,
+      "tokens_seen": 2436431872
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2856684320333815e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.403917615200408,
+      "tokens_seen": 2436562944
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.284865992617558e-05,
+      "loss": 1.3438,
+      "theoretical_loss": 3.4039030571795905,
+      "tokens_seen": 2436694016
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2840635532017334e-05,
+      "loss": 1.29,
+      "theoretical_loss": 3.4038885001610923,
+      "tokens_seen": 2436825088
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.283261113785909e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.403873944144791,
+      "tokens_seen": 2436956160
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2824586743700854e-05,
+      "loss": 1.2767,
+      "theoretical_loss": 3.403859389130564,
+      "tokens_seen": 2437087232
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.281656234954261e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.4038448351182877,
+      "tokens_seen": 2437218304
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2808537955384374e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.4038302821078394,
+      "tokens_seen": 2437349376
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.280051356122613e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.403815730099097,
+      "tokens_seen": 2437480448
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2792489167067894e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.403801179091937,
+      "tokens_seen": 2437611520
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.278446477290965e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4037866290862366,
+      "tokens_seen": 2437742592
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.27764403787514e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.403772080081873,
+      "tokens_seen": 2437873664
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.276841598459317e-05,
+      "loss": 1.2604,
+      "theoretical_loss": 3.403757532078724,
+      "tokens_seen": 2438004736
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.276039159043492e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.4037429850766667,
+      "tokens_seen": 2438135808
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.275236719627669e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.4037284390755778,
+      "tokens_seen": 2438266880
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.274434280211844e-05,
+      "loss": 1.2599,
+      "theoretical_loss": 3.4037138940753353,
+      "tokens_seen": 2438397952
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2736318407960206e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.403699350075817,
+      "tokens_seen": 2438529024
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.272829401380196e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.4036848070768984,
+      "tokens_seen": 2438660096
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.272026961964371e-05,
+      "loss": 1.2721,
+      "theoretical_loss": 3.4036702650784587,
+      "tokens_seen": 2438791168
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.271224522548548e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.403655724080375,
+      "tokens_seen": 2438922240
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.270422083132723e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.4036411840825234,
+      "tokens_seen": 2439053312
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2696196437169e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.403626645084783,
+      "tokens_seen": 2439184384
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.268817204301075e-05,
+      "loss": 1.3408,
+      "theoretical_loss": 3.4036121070870307,
+      "tokens_seen": 2439315456
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.268014764885252e-05,
+      "loss": 1.3043,
+      "theoretical_loss": 3.4035975700891434,
+      "tokens_seen": 2439446528
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.267212325469427e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.4035830340909996,
+      "tokens_seen": 2439577600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.266409886053604e-05,
+      "loss": 1.1781,
+      "theoretical_loss": 3.4035684990924766,
+      "tokens_seen": 2439708672
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.265607446637779e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.403553965093451,
+      "tokens_seen": 2439839744
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2648050072219545e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.4035394320938015,
+      "tokens_seen": 2439970816
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.264002567806131e-05,
+      "loss": 1.2602,
+      "theoretical_loss": 3.4035249000934056,
+      "tokens_seen": 2440101888
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2632001283903065e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.40351036909214,
+      "tokens_seen": 2440232960
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.262397688974483e-05,
+      "loss": 1.2746,
+      "theoretical_loss": 3.4034958390898833,
+      "tokens_seen": 2440364032
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2615952495586585e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.403481310086513,
+      "tokens_seen": 2440495104
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.260792810142835e-05,
+      "loss": 1.1999,
+      "theoretical_loss": 3.4034667820819067,
+      "tokens_seen": 2440626176
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2599903707270105e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.403452255075942,
+      "tokens_seen": 2440757248
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.259187931311186e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.4034377290684965,
+      "tokens_seen": 2440888320
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2583854918953624e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.403423204059448,
+      "tokens_seen": 2441019392
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.257583052479538e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.4034086800486745,
+      "tokens_seen": 2441150464
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2567806130637144e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.403394157036054,
+      "tokens_seen": 2441281536
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.25597817364789e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.4033796350214636,
+      "tokens_seen": 2441412608
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2551757342320664e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.4033651140047816,
+      "tokens_seen": 2441543680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.254373294816242e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.403350593985886,
+      "tokens_seen": 2441674752
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.253570855400417e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4033360749646544,
+      "tokens_seen": 2441805824
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.252768415984594e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.403321556940965,
+      "tokens_seen": 2441936896
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.251965976568769e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.4033070399146954,
+      "tokens_seen": 2442067968
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.251163537152946e-05,
+      "loss": 1.1755,
+      "theoretical_loss": 3.4032925238857237,
+      "tokens_seen": 2442199040
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.250361097737121e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.4032780088539276,
+      "tokens_seen": 2442330112
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2495586583212977e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.4032634948191856,
+      "tokens_seen": 2442461184
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.248756218905473e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.4032489817813754,
+      "tokens_seen": 2442592256
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.247953779489648e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.4032344697403754,
+      "tokens_seen": 2442723328
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.247151340073825e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.403219958696063,
+      "tokens_seen": 2442854400
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 820144,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2366188764572144,
+      "objective/train/theoretical_loss": 3.4032054486483165,
+      "objective/train/tokens_used": 813444576,
+      "theoretical_loss": 3.4032054486483165,
+      "tokens_seen": 2442985472
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.246348900658e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4032054486483165,
+      "tokens_seen": 2442985472
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.245546461242177e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.403190939597014,
+      "tokens_seen": 2443116544
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.244744021826352e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4031764315420343,
+      "tokens_seen": 2443247616
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.243941582410529e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.4031619244832543,
+      "tokens_seen": 2443378688
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.243139142994704e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.403147418420553,
+      "tokens_seen": 2443509760
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2423367035788795e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.403132913353809,
+      "tokens_seen": 2443640832
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.241534264163056e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.403118409282899,
+      "tokens_seen": 2443771904
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2407318247472315e-05,
+      "loss": 1.3014,
+      "theoretical_loss": 3.4031039062077024,
+      "tokens_seen": 2443902976
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.239929385331408e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.4030894041280977,
+      "tokens_seen": 2444034048
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2391269459155835e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.4030749030439624,
+      "tokens_seen": 2444165120
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.23832450649976e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.4030604029551745,
+      "tokens_seen": 2444296192
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2375220670839355e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.4030459038616137,
+      "tokens_seen": 2444427264
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.236719627668112e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4030314057631568,
+      "tokens_seen": 2444558336
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2359171882522875e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.403016908659683,
+      "tokens_seen": 2444689408
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.235114748836463e-05,
+      "loss": 1.1973,
+      "theoretical_loss": 3.4030024125510705,
+      "tokens_seen": 2444820480
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2343123094206394e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.4029879174371978,
+      "tokens_seen": 2444951552
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.233509870004815e-05,
+      "loss": 1.2818,
+      "theoretical_loss": 3.4029734233179436,
+      "tokens_seen": 2445082624
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2327074305889914e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.4029589301931855,
+      "tokens_seen": 2445213696
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.231904991173167e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.4029444380628027,
+      "tokens_seen": 2445344768
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2311025517573434e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.402929946926673,
+      "tokens_seen": 2445475840
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.230300112341519e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.4029154567846756,
+      "tokens_seen": 2445606912
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.229497672925694e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.402900967636689,
+      "tokens_seen": 2445737984
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.228695233509871e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.4028864794825906,
+      "tokens_seen": 2445869056
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.227892794094046e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.4028719923222606,
+      "tokens_seen": 2446000128
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.227090354678223e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.4028575061555766,
+      "tokens_seen": 2446131200
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.226287915262398e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.4028430209824174,
+      "tokens_seen": 2446262272
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.225485475846575e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.402828536802662,
+      "tokens_seen": 2446393344
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.22468303643075e-05,
+      "loss": 1.3028,
+      "theoretical_loss": 3.402814053616188,
+      "tokens_seen": 2446524416
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.223880597014925e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4027995714228756,
+      "tokens_seen": 2446655488
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.223078157599102e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.4027850902226024,
+      "tokens_seen": 2446786560
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.222275718183277e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.4027706100152475,
+      "tokens_seen": 2446917632
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.221473278767454e-05,
+      "loss": 1.1972,
+      "theoretical_loss": 3.4027561308006895,
+      "tokens_seen": 2447048704
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.220670839351629e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.4027416525788072,
+      "tokens_seen": 2447179776
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.219868399935805e-05,
+      "loss": 1.2795,
+      "theoretical_loss": 3.4027271753494794,
+      "tokens_seen": 2447310848
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.219065960519981e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.402712699112585,
+      "tokens_seen": 2447441920
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2182635211041565e-05,
+      "loss": 1.2996,
+      "theoretical_loss": 3.402698223868003,
+      "tokens_seen": 2447572992
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.217461081688333e-05,
+      "loss": 1.1486,
+      "theoretical_loss": 3.4026837496156115,
+      "tokens_seen": 2447704064
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2166586422725085e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.40266927635529,
+      "tokens_seen": 2447835136
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.215856202856685e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.402654804086917,
+      "tokens_seen": 2447966208
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2150537634408605e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.402640332810372,
+      "tokens_seen": 2448097280
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2142513240250365e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.4026258625255332,
+      "tokens_seen": 2448228352
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2134488846092125e-05,
+      "loss": 1.2111,
+      "theoretical_loss": 3.40261139323228,
+      "tokens_seen": 2448359424
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.212646445193388e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.4025969249304913,
+      "tokens_seen": 2448490496
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.2118440057775645e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.4025824576200465,
+      "tokens_seen": 2448621568
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.21104156636174e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.4025679913008235,
+      "tokens_seen": 2448752640
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.210239126945916e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.4025535259727024,
+      "tokens_seen": 2448883712
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.209436687530092e-05,
+      "loss": 1.1676,
+      "theoretical_loss": 3.402539061635562,
+      "tokens_seen": 2449014784
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.208634248114268e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.402524598289281,
+      "tokens_seen": 2449145856
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.207831808698444e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.402510135933739,
+      "tokens_seen": 2449276928
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.207029369282619e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.402495674568815,
+      "tokens_seen": 2449408000
+    },
+    {
+      "epoch": 0.48,
+      "objective/train/docs_used": 822421,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2862545251846313,
+      "objective/train/theoretical_loss": 3.4024812141943874,
+      "objective/train/tokens_used": 819998176,
+      "theoretical_loss": 3.4024812141943874,
+      "tokens_seen": 2449539072
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.206226929866795e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.4024812141943874,
+      "tokens_seen": 2449539072
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.205424490450971e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4024667548103364,
+      "tokens_seen": 2449670144
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.204622051035147e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.402452296416541,
+      "tokens_seen": 2449801216
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.203819611619323e-05,
+      "loss": 1.1465,
+      "theoretical_loss": 3.4024378390128796,
+      "tokens_seen": 2449932288
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.203017172203499e-05,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4024233825992325,
+      "tokens_seen": 2450063360
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 5.202214732787675e-05,
+      "loss": 1.2154,
+      "theoretical_loss": 3.4024089271754785,
+      "tokens_seen": 2450194432
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.201412293371851e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.4023944727414968,
+      "tokens_seen": 2450325504
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.200609853956026e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.402380019297166,
+      "tokens_seen": 2450456576
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.199807414540202e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.402365566842367,
+      "tokens_seen": 2450587648
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.199004975124378e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.4023511153769777,
+      "tokens_seen": 2450718720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.198202535708554e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.4023366649008784,
+      "tokens_seen": 2450849792
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.19740009629273e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.402322215413948,
+      "tokens_seen": 2450980864
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1965976568769056e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.402307766916066,
+      "tokens_seen": 2451111936
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.195795217461082e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.4022933194071117,
+      "tokens_seen": 2451243008
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1949927780452576e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.4022788728869644,
+      "tokens_seen": 2451374080
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1941903386294336e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.4022644273555045,
+      "tokens_seen": 2451505152
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1933878992136095e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.4022499828126103,
+      "tokens_seen": 2451636224
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1925854597977855e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.402235539258162,
+      "tokens_seen": 2451767296
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1917830203819615e-05,
+      "loss": 1.3059,
+      "theoretical_loss": 3.4022210966920383,
+      "tokens_seen": 2451898368
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.190980580966137e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.40220665511412,
+      "tokens_seen": 2452029440
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1901781415503135e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.4021922145242858,
+      "tokens_seen": 2452160512
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.189375702134489e-05,
+      "loss": 1.3113,
+      "theoretical_loss": 3.4021777749224156,
+      "tokens_seen": 2452291584
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.188573262718665e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.4021633363083885,
+      "tokens_seen": 2452422656
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.187770823302841e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.402148898682085,
+      "tokens_seen": 2452553728
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.186968383887016e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.402134462043384,
+      "tokens_seen": 2452684800
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.186165944471193e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.402120026392166,
+      "tokens_seen": 2452815872
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.185363505055368e-05,
+      "loss": 1.2731,
+      "theoretical_loss": 3.4021055917283096,
+      "tokens_seen": 2452946944
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.184561065639545e-05,
+      "loss": 1.1879,
+      "theoretical_loss": 3.4020911580516953,
+      "tokens_seen": 2453078016
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.18375862622372e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.4020767253622024,
+      "tokens_seen": 2453209088
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.182956186807896e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.402062293659711,
+      "tokens_seen": 2453340160
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.182153747392072e-05,
+      "loss": 1.2522,
+      "theoretical_loss": 3.4020478629441007,
+      "tokens_seen": 2453471232
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1813513079762474e-05,
+      "loss": 1.321,
+      "theoretical_loss": 3.4020334332152515,
+      "tokens_seen": 2453602304
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.180548868560424e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4020190044730425,
+      "tokens_seen": 2453733376
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1797464291445994e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.4020045767173546,
+      "tokens_seen": 2453864448
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.178943989728776e-05,
+      "loss": 1.2848,
+      "theoretical_loss": 3.4019901499480674,
+      "tokens_seen": 2453995520
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.178141550312951e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.40197572416506,
+      "tokens_seen": 2454126592
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1773391108971266e-05,
+      "loss": 1.3419,
+      "theoretical_loss": 3.401961299368213,
+      "tokens_seen": 2454257664
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.176536671481303e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.4019468755574063,
+      "tokens_seen": 2454388736
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1757342320654786e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4019324527325194,
+      "tokens_seen": 2454519808
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.174931792649655e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.401918030893433,
+      "tokens_seen": 2454650880
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1741293532338306e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.401903610040026,
+      "tokens_seen": 2454781952
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.173326913818007e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.4018891901721795,
+      "tokens_seen": 2454913024
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1725244744021826e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.4018747712897732,
+      "tokens_seen": 2455044096
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.171722034986359e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.4018603533926868,
+      "tokens_seen": 2455175168
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1709195955705346e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.4018459364808007,
+      "tokens_seen": 2455306240
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.17011715615471e-05,
+      "loss": 1.3477,
+      "theoretical_loss": 3.401831520553995,
+      "tokens_seen": 2455437312
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1693147167388866e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.4018171056121496,
+      "tokens_seen": 2455568384
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.168512277323062e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.4018026916551447,
+      "tokens_seen": 2455699456
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1677098379072385e-05,
+      "loss": 1.3088,
+      "theoretical_loss": 3.4017882786828606,
+      "tokens_seen": 2455830528
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.166907398491414e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.4017738666951773,
+      "tokens_seen": 2455961600
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 824774,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2480775117874146,
+      "objective/train/theoretical_loss": 3.401759455691975,
+      "objective/train/tokens_used": 826551776,
+      "theoretical_loss": 3.401759455691975,
+      "tokens_seen": 2456092672
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1661049590755905e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.401759455691975,
+      "tokens_seen": 2456092672
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.165302519659766e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.401745045673134,
+      "tokens_seen": 2456223744
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.164500080243941e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.401730636638535,
+      "tokens_seen": 2456354816
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.163697640828118e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.4017162285880573,
+      "tokens_seen": 2456485888
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.162895201412293e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.4017018215215815,
+      "tokens_seen": 2456616960
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.16209276199647e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4016874154389884,
+      "tokens_seen": 2456748032
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.161290322580645e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.4016730103401582,
+      "tokens_seen": 2456879104
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.160487883164822e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.401658606224971,
+      "tokens_seen": 2457010176
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.159685443748997e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.4016442030933067,
+      "tokens_seen": 2457141248
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1588830043331724e-05,
+      "loss": 1.3143,
+      "theoretical_loss": 3.4016298009450465,
+      "tokens_seen": 2457272320
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.158080564917349e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.4016153997800704,
+      "tokens_seen": 2457403392
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1572781255015244e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.4016009995982595,
+      "tokens_seen": 2457534464
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.156475686085701e-05,
+      "loss": 1.2993,
+      "theoretical_loss": 3.401586600399493,
+      "tokens_seen": 2457665536
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1556732466698764e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.4015722021836523,
+      "tokens_seen": 2457796608
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.154870807254053e-05,
+      "loss": 1.2611,
+      "theoretical_loss": 3.4015578049506177,
+      "tokens_seen": 2457927680
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1540683678382283e-05,
+      "loss": 1.2959,
+      "theoretical_loss": 3.4015434087002694,
+      "tokens_seen": 2458058752
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1532659284224037e-05,
+      "loss": 1.2812,
+      "theoretical_loss": 3.401529013432488,
+      "tokens_seen": 2458189824
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.15246348900658e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.4015146191471546,
+      "tokens_seen": 2458320896
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1516610495907556e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4015002258441496,
+      "tokens_seen": 2458451968
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.150858610174932e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.401485833523353,
+      "tokens_seen": 2458583040
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1500561707591076e-05,
+      "loss": 1.2832,
+      "theoretical_loss": 3.401471442184646,
+      "tokens_seen": 2458714112
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.149253731343284e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.401457051827909,
+      "tokens_seen": 2458845184
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1484512919274596e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4014426624530225,
+      "tokens_seen": 2458976256
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.147648852511635e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4014282740598674,
+      "tokens_seen": 2459107328
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1468464130958116e-05,
+      "loss": 1.2651,
+      "theoretical_loss": 3.401413886648325,
+      "tokens_seen": 2459238400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.146043973679987e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.4013995002182744,
+      "tokens_seen": 2459369472
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1452415342641636e-05,
+      "loss": 1.287,
+      "theoretical_loss": 3.401385114769598,
+      "tokens_seen": 2459500544
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.144439094848339e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.401370730302176,
+      "tokens_seen": 2459631616
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1436366554325155e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.4013563468158887,
+      "tokens_seen": 2459762688
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.142834216016691e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.4013419643106175,
+      "tokens_seen": 2459893760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1420317766008675e-05,
+      "loss": 1.1698,
+      "theoretical_loss": 3.401327582786243,
+      "tokens_seen": 2460024832
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.141229337185043e-05,
+      "loss": 1.1934,
+      "theoretical_loss": 3.4013132022426458,
+      "tokens_seen": 2460155904
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.140426897769218e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.401298822679707,
+      "tokens_seen": 2460286976
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.139624458353395e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.401284444097308,
+      "tokens_seen": 2460418048
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.13882201893757e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.4012700664953286,
+      "tokens_seen": 2460549120
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.138019579521747e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.401255689873651,
+      "tokens_seen": 2460680192
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.137217140105922e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.401241314232155,
+      "tokens_seen": 2460811264
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.136414700690099e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.4012269395707224,
+      "tokens_seen": 2460942336
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.135612261274274e-05,
+      "loss": 1.3215,
+      "theoretical_loss": 3.401212565889234,
+      "tokens_seen": 2461073408
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1348098218584494e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.40119819318757,
+      "tokens_seen": 2461204480
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.134007382442626e-05,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4011838214656125,
+      "tokens_seen": 2461335552
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1332049430268014e-05,
+      "loss": 1.3202,
+      "theoretical_loss": 3.4011694507232417,
+      "tokens_seen": 2461466624
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.132402503610978e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.40115508096034,
+      "tokens_seen": 2461597696
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1316000641951534e-05,
+      "loss": 1.177,
+      "theoretical_loss": 3.4011407121767867,
+      "tokens_seen": 2461728768
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.13079762477933e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.4011263443724644,
+      "tokens_seen": 2461859840
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1299951853635053e-05,
+      "loss": 1.3066,
+      "theoretical_loss": 3.401111977547253,
+      "tokens_seen": 2461990912
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1291927459476807e-05,
+      "loss": 1.1641,
+      "theoretical_loss": 3.401097611701035,
+      "tokens_seen": 2462121984
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.128390306531857e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.4010832468336907,
+      "tokens_seen": 2462253056
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1275878671160326e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.4010688829451015,
+      "tokens_seen": 2462384128
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.126785427700209e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4010545200351485,
+      "tokens_seen": 2462515200
+    },
+    {
+      "epoch": 0.49,
+      "objective/train/docs_used": 827146,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3098605871200562,
+      "objective/train/theoretical_loss": 3.4010401581037133,
+      "objective/train/tokens_used": 833105376,
+      "theoretical_loss": 3.4010401581037133,
+      "tokens_seen": 2462646272
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1259829882843846e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.4010401581037133,
+      "tokens_seen": 2462646272
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.125180548868561e-05,
+      "loss": 1.2985,
+      "theoretical_loss": 3.4010257971506768,
+      "tokens_seen": 2462777344
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1243781094527366e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.4010114371759204,
+      "tokens_seen": 2462908416
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.123575670036912e-05,
+      "loss": 1.3027,
+      "theoretical_loss": 3.4009970781793255,
+      "tokens_seen": 2463039488
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1227732306210886e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.4009827201607727,
+      "tokens_seen": 2463170560
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.121970791205264e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.4009683631201444,
+      "tokens_seen": 2463301632
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1211683517894406e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.400954007057322,
+      "tokens_seen": 2463432704
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.120365912373616e-05,
+      "loss": 1.2956,
+      "theoretical_loss": 3.4009396519721857,
+      "tokens_seen": 2463563776
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1195634729577925e-05,
+      "loss": 1.179,
+      "theoretical_loss": 3.400925297864618,
+      "tokens_seen": 2463694848
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.118761033541968e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.4009109447344996,
+      "tokens_seen": 2463825920
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.117958594126143e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.4008965925817125,
+      "tokens_seen": 2463956992
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.11715615471032e-05,
+      "loss": 1.1844,
+      "theoretical_loss": 3.400882241406138,
+      "tokens_seen": 2464088064
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.116353715294495e-05,
+      "loss": 1.2259,
+      "theoretical_loss": 3.4008678912076573,
+      "tokens_seen": 2464219136
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.115551275878672e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4008535419861525,
+      "tokens_seen": 2464350208
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.114748836462847e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.4008391937415046,
+      "tokens_seen": 2464481280
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.113946397047024e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.4008248464735953,
+      "tokens_seen": 2464612352
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.113143957631199e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.400810500182306,
+      "tokens_seen": 2464743424
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1123415182153744e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.400796154867519,
+      "tokens_seen": 2464874496
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.111539078799551e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.400781810529115,
+      "tokens_seen": 2465005568
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1107366393837264e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.4007674671669763,
+      "tokens_seen": 2465136640
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.109934199967903e-05,
+      "loss": 1.3204,
+      "theoretical_loss": 3.400753124780984,
+      "tokens_seen": 2465267712
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1091317605520784e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.40073878337102,
+      "tokens_seen": 2465398784
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.108329321136255e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.4007244429369665,
+      "tokens_seen": 2465529856
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1075268817204304e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.4007101034787044,
+      "tokens_seen": 2465660928
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.106724442304607e-05,
+      "loss": 1.1677,
+      "theoretical_loss": 3.400695764996116,
+      "tokens_seen": 2465792000
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1059220028887824e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.400681427489083,
+      "tokens_seen": 2465923072
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.105119563472958e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.4006670909574868,
+      "tokens_seen": 2466054144
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.104317124057134e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.400652755401209,
+      "tokens_seen": 2466185216
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1035146846413096e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.400638420820133,
+      "tokens_seen": 2466316288
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.102712245225486e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.4006240872141387,
+      "tokens_seen": 2466447360
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.1019098058096616e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4006097545831087,
+      "tokens_seen": 2466578432
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 5.101107366393838e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.400595422926925,
+      "tokens_seen": 2466709504
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.1003049269780136e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.40058109224547,
+      "tokens_seen": 2466840576
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.099502487562189e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.400566762538624,
+      "tokens_seen": 2466971648
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0987000481463656e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.4005524338062707,
+      "tokens_seen": 2467102720
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.097897608730541e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.400538106048291,
+      "tokens_seen": 2467233792
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0970951693147176e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.4005237792645677,
+      "tokens_seen": 2467364864
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.096292729898893e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4005094534549825,
+      "tokens_seen": 2467495936
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0954902904830696e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.4004951286194167,
+      "tokens_seen": 2467627008
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.094687851067245e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.400480804757753,
+      "tokens_seen": 2467758080
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.09388541165142e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.4004664818698735,
+      "tokens_seen": 2467889152
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.093082972235597e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.4004521599556603,
+      "tokens_seen": 2468020224
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.092280532819772e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.400437839014995,
+      "tokens_seen": 2468151296
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.091478093403949e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.4004235190477603,
+      "tokens_seen": 2468282368
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.090675653988124e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.400409200053838,
+      "tokens_seen": 2468413440
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.089873214572301e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.400394882033111,
+      "tokens_seen": 2468544512
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.089070775156476e-05,
+      "loss": 1.15,
+      "theoretical_loss": 3.4003805649854604,
+      "tokens_seen": 2468675584
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0882683357406514e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.400366248910769,
+      "tokens_seen": 2468806656
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.087465896324828e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.4003519338089188,
+      "tokens_seen": 2468937728
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0866634569090034e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.4003376196797923,
+      "tokens_seen": 2469068800
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 829005,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.131378412246704,
+      "objective/train/theoretical_loss": 3.4003233065232714,
+      "objective/train/tokens_used": 839658976,
+      "theoretical_loss": 3.4003233065232714,
+      "tokens_seen": 2469199872
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.08586101749318e-05,
+      "loss": 1.1491,
+      "theoretical_loss": 3.4003233065232714,
+      "tokens_seen": 2469199872
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0850585780773554e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.400308994339239,
+      "tokens_seen": 2469330944
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0842561386615314e-05,
+      "loss": 1.3102,
+      "theoretical_loss": 3.400294683127577,
+      "tokens_seen": 2469462016
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0834536992457074e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.400280372888168,
+      "tokens_seen": 2469593088
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.082651259829883e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.4002660636208937,
+      "tokens_seen": 2469724160
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0818488204140594e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.4002517553256366,
+      "tokens_seen": 2469855232
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.081046380998235e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.4002374480022803,
+      "tokens_seen": 2469986304
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0802439415824113e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.400223141650706,
+      "tokens_seen": 2470117376
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0794415021665867e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4002088362707963,
+      "tokens_seen": 2470248448
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0786390627507626e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.400194531862434,
+      "tokens_seen": 2470379520
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0778366233349386e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.4001802284255014,
+      "tokens_seen": 2470510592
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0770341839191146e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.4001659259598807,
+      "tokens_seen": 2470641664
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0762317445032906e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.4001516244654546,
+      "tokens_seen": 2470772736
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.075429305087466e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.400137323942106,
+      "tokens_seen": 2470903808
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.074626865671642e-05,
+      "loss": 1.1935,
+      "theoretical_loss": 3.4001230243897176,
+      "tokens_seen": 2471034880
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.073824426255818e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.400108725808171,
+      "tokens_seen": 2471165952
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.073021986839994e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.4000944281973493,
+      "tokens_seen": 2471297024
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.07221954742417e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4000801315571354,
+      "tokens_seen": 2471428096
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.071417108008346e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.400065835887412,
+      "tokens_seen": 2471559168
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.070614668592522e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.400051541188061,
+      "tokens_seen": 2471690240
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.069812229176697e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.4000372474589655,
+      "tokens_seen": 2471821312
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.069009789760873e-05,
+      "loss": 1.3218,
+      "theoretical_loss": 3.4000229547000087,
+      "tokens_seen": 2471952384
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.068207350345049e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.400008662911073,
+      "tokens_seen": 2472083456
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.067404910929225e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.3999943720920403,
+      "tokens_seen": 2472214528
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.066602471513401e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3999800822427946,
+      "tokens_seen": 2472345600
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.065800032097577e-05,
+      "loss": 1.1769,
+      "theoretical_loss": 3.399965793363218,
+      "tokens_seen": 2472476672
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0649975926817525e-05,
+      "loss": 1.1958,
+      "theoretical_loss": 3.3999515054531937,
+      "tokens_seen": 2472607744
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0641951532659284e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.399937218512604,
+      "tokens_seen": 2472738816
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0633927138501044e-05,
+      "loss": 1.2646,
+      "theoretical_loss": 3.3999229325413323,
+      "tokens_seen": 2472869888
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0625902744342804e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.399908647539261,
+      "tokens_seen": 2473000960
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0617878350184564e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.399894363506273,
+      "tokens_seen": 2473132032
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0609853956026324e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3998800804422515,
+      "tokens_seen": 2473263104
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0601829561868084e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.3998657983470792,
+      "tokens_seen": 2473394176
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.059380516770984e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.3998515172206396,
+      "tokens_seen": 2473525248
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.05857807735516e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.399837237062815,
+      "tokens_seen": 2473656320
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.057775637939336e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3998229578734884,
+      "tokens_seen": 2473787392
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.056973198523512e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3998086796525433,
+      "tokens_seen": 2473918464
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.056170759107688e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.3997944023998623,
+      "tokens_seen": 2474049536
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.055368319691863e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.399780126115328,
+      "tokens_seen": 2474180608
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0545658802760397e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.399765850798825,
+      "tokens_seen": 2474311680
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.053763440860215e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.399751576450235,
+      "tokens_seen": 2474442752
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.052961001444391e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.3997373030694416,
+      "tokens_seen": 2474573824
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.052158562028567e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3997230306563275,
+      "tokens_seen": 2474704896
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.051356122612743e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3997087592107764,
+      "tokens_seen": 2474835968
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.050553683196919e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3996944887326714,
+      "tokens_seen": 2474967040
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.049751243781094e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.399680219221896,
+      "tokens_seen": 2475098112
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.048948804365271e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.399665950678332,
+      "tokens_seen": 2475229184
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.048146364949446e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3996516831018644,
+      "tokens_seen": 2475360256
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.047343925533623e-05,
+      "loss": 1.1426,
+      "theoretical_loss": 3.399637416492375,
+      "tokens_seen": 2475491328
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.046541486117798e-05,
+      "loss": 1.1898,
+      "theoretical_loss": 3.3996231508497483,
+      "tokens_seen": 2475622400
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 831408,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.27910315990448,
+      "objective/train/theoretical_loss": 3.399608886173867,
+      "objective/train/tokens_used": 846212576,
+      "theoretical_loss": 3.399608886173867,
+      "tokens_seen": 2475753472
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0457390467019735e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.399608886173867,
+      "tokens_seen": 2475753472
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.04493660728615e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.399594622464614,
+      "tokens_seen": 2475884544
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0441341678703255e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.399580359721873,
+      "tokens_seen": 2476015616
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.043331728454502e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3995660979455278,
+      "tokens_seen": 2476146688
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0425292890386775e-05,
+      "loss": 1.2105,
+      "theoretical_loss": 3.399551837135461,
+      "tokens_seen": 2476277760
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.041726849622854e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.3995375772915564,
+      "tokens_seen": 2476408832
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0409244102070295e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3995233184136975,
+      "tokens_seen": 2476539904
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.040121970791205e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.3995090605017677,
+      "tokens_seen": 2476670976
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0393195313753814e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.3994948035556503,
+      "tokens_seen": 2476802048
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.038517091959557e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.3994805475752288,
+      "tokens_seen": 2476933120
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0377146525437334e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.399466292560387,
+      "tokens_seen": 2477064192
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.036912213127909e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.3994520385110074,
+      "tokens_seen": 2477195264
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0361097737120854e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.399437785426975,
+      "tokens_seen": 2477326336
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.035307334296261e-05,
+      "loss": 1.1892,
+      "theoretical_loss": 3.3994235333081724,
+      "tokens_seen": 2477457408
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.034504894880436e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.3994092821544832,
+      "tokens_seen": 2477588480
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.033702455464613e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.399395031965791,
+      "tokens_seen": 2477719552
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.032900016048788e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.39938078274198,
+      "tokens_seen": 2477850624
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.032097576632965e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.399366534482933,
+      "tokens_seen": 2477981696
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.03129513721714e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3993522871885347,
+      "tokens_seen": 2478112768
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.030492697801317e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3993380408586678,
+      "tokens_seen": 2478243840
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.029690258385492e-05,
+      "loss": 1.2021,
+      "theoretical_loss": 3.399323795493216,
+      "tokens_seen": 2478374912
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.028887818969667e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3993095510920637,
+      "tokens_seen": 2478505984
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.028085379553844e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.3992953076550942,
+      "tokens_seen": 2478637056
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.027282940138019e-05,
+      "loss": 1.175,
+      "theoretical_loss": 3.3992810651821914,
+      "tokens_seen": 2478768128
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.026480500722196e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.3992668236732393,
+      "tokens_seen": 2478899200
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.025678061306371e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.399252583128121,
+      "tokens_seen": 2479030272
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.024875621890548e-05,
+      "loss": 1.1698,
+      "theoretical_loss": 3.399238343546721,
+      "tokens_seen": 2479161344
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.024073182474723e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.3992241049289227,
+      "tokens_seen": 2479292416
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0232707430588985e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.3992098672746103,
+      "tokens_seen": 2479423488
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.022468303643075e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3991956305836672,
+      "tokens_seen": 2479554560
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0216658642272505e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3991813948559777,
+      "tokens_seen": 2479685632
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.020863424811427e-05,
+      "loss": 1.1916,
+      "theoretical_loss": 3.3991671600914257,
+      "tokens_seen": 2479816704
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0200609853956025e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.3991529262898954,
+      "tokens_seen": 2479947776
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.019258545979779e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.3991386934512695,
+      "tokens_seen": 2480078848
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0184561065639545e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3991244615754335,
+      "tokens_seen": 2480209920
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.01765366714813e-05,
+      "loss": 1.2111,
+      "theoretical_loss": 3.399110230662271,
+      "tokens_seen": 2480340992
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0168512277323065e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3990960007116655,
+      "tokens_seen": 2480472064
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.016048788316482e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.399081771723501,
+      "tokens_seen": 2480603136
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0152463489006585e-05,
+      "loss": 1.152,
+      "theoretical_loss": 3.399067543697662,
+      "tokens_seen": 2480734208
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.014443909484834e-05,
+      "loss": 1.2314,
+      "theoretical_loss": 3.3990533166340327,
+      "tokens_seen": 2480865280
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0136414700690104e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.399039090532497,
+      "tokens_seen": 2480996352
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.012839030653186e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3990248653929385,
+      "tokens_seen": 2481127424
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.0120365912373624e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.3990106412152423,
+      "tokens_seen": 2481258496
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.011234151821538e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.398996417999292,
+      "tokens_seen": 2481389568
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.010431712405713e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.3989821957449715,
+      "tokens_seen": 2481520640
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.00962927298989e-05,
+      "loss": 1.3152,
+      "theoretical_loss": 3.398967974452166,
+      "tokens_seen": 2481651712
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.008826833574065e-05,
+      "loss": 1.3335,
+      "theoretical_loss": 3.3989537541207584,
+      "tokens_seen": 2481782784
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.008024394158242e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.398939534750634,
+      "tokens_seen": 2481913856
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.007221954742417e-05,
+      "loss": 1.1814,
+      "theoretical_loss": 3.398925316341676,
+      "tokens_seen": 2482044928
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.006419515326594e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.39891109889377,
+      "tokens_seen": 2482176000
+    },
+    {
+      "epoch": 0.5,
+      "objective/train/docs_used": 833535,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2446690797805786,
+      "objective/train/theoretical_loss": 3.3988968824067998,
+      "objective/train/tokens_used": 852766176,
+      "theoretical_loss": 3.3988968824067998,
+      "tokens_seen": 2482307072
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.005617075910769e-05,
+      "loss": 1.167,
+      "theoretical_loss": 3.3988968824067998,
+      "tokens_seen": 2482307072
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.004814636494944e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.398882666880649,
+      "tokens_seen": 2482438144
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.004012197079121e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3988684523152024,
+      "tokens_seen": 2482569216
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.003209757663296e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.398854238710345,
+      "tokens_seen": 2482700288
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.002407318247473e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.39884002606596,
+      "tokens_seen": 2482831360
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.001604878831648e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3988258143819325,
+      "tokens_seen": 2482962432
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 5.000802439415825e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3988116036581473,
+      "tokens_seen": 2483093504
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 5e-05,
+      "loss": 1.3044,
+      "theoretical_loss": 3.3987973938944878,
+      "tokens_seen": 2483224576
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.999197560584176e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.3987831850908394,
+      "tokens_seen": 2483355648
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.998395121168352e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.398768977247086,
+      "tokens_seen": 2483486720
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.997592681752528e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3987547703631122,
+      "tokens_seen": 2483617792
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9967902423367035e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.398740564438803,
+      "tokens_seen": 2483748864
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9959878029208795e-05,
+      "loss": 1.313,
+      "theoretical_loss": 3.3987263594740424,
+      "tokens_seen": 2483879936
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9951853635050555e-05,
+      "loss": 1.1748,
+      "theoretical_loss": 3.398712155468715,
+      "tokens_seen": 2484011008
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9943829240892315e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3986979524227054,
+      "tokens_seen": 2484142080
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9935804846734075e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.3986837503358984,
+      "tokens_seen": 2484273152
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9927780452575835e-05,
+      "loss": 1.3104,
+      "theoretical_loss": 3.3986695492081784,
+      "tokens_seen": 2484404224
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9919756058417595e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3986553490394305,
+      "tokens_seen": 2484535296
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9911731664259355e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.3986411498295386,
+      "tokens_seen": 2484666368
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.990370727010111e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.398626951578388,
+      "tokens_seen": 2484797440
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.989568287594287e-05,
+      "loss": 1.2602,
+      "theoretical_loss": 3.398612754285863,
+      "tokens_seen": 2484928512
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.988765848178463e-05,
+      "loss": 1.229,
+      "theoretical_loss": 3.3985985579518485,
+      "tokens_seen": 2485059584
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.987963408762639e-05,
+      "loss": 1.1908,
+      "theoretical_loss": 3.398584362576229,
+      "tokens_seen": 2485190656
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.987160969346815e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.39857016815889,
+      "tokens_seen": 2485321728
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.986358529930991e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.3985559746997156,
+      "tokens_seen": 2485452800
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.985556090515167e-05,
+      "loss": 1.0953,
+      "theoretical_loss": 3.3985417821985906,
+      "tokens_seen": 2485583872
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.984753651099342e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.3985275906553998,
+      "tokens_seen": 2485714944
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.983951211683518e-05,
+      "loss": 1.1227,
+      "theoretical_loss": 3.3985134000700286,
+      "tokens_seen": 2485846016
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.983148772267694e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3984992104423615,
+      "tokens_seen": 2485977088
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.98234633285187e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.3984850217722826,
+      "tokens_seen": 2486108160
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.981543893436046e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.3984708340596783,
+      "tokens_seen": 2486239232
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.980741454020222e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3984566473044326,
+      "tokens_seen": 2486370304
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.979939014604398e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.3984424615064306,
+      "tokens_seen": 2486501376
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.979136575188573e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.398428276665557,
+      "tokens_seen": 2486632448
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.978334135772749e-05,
+      "loss": 1.2815,
+      "theoretical_loss": 3.398414092781697,
+      "tokens_seen": 2486763520
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.977531696356925e-05,
+      "loss": 1.1713,
+      "theoretical_loss": 3.398399909854736,
+      "tokens_seen": 2486894592
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.976729256941101e-05,
+      "loss": 1.3061,
+      "theoretical_loss": 3.398385727884558,
+      "tokens_seen": 2487025664
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.975926817525277e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3983715468710494,
+      "tokens_seen": 2487156736
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.975124378109453e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.398357366814094,
+      "tokens_seen": 2487287808
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.974321938693629e-05,
+      "loss": 1.1722,
+      "theoretical_loss": 3.3983431877135777,
+      "tokens_seen": 2487418880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.973519499277805e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.3983290095693848,
+      "tokens_seen": 2487549952
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9727170598619805e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.398314832381401,
+      "tokens_seen": 2487681024
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9719146204461565e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.398300656149512,
+      "tokens_seen": 2487812096
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9711121810303325e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.398286480873602,
+      "tokens_seen": 2487943168
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9703097416145085e-05,
+      "loss": 1.2072,
+      "theoretical_loss": 3.3982723065535563,
+      "tokens_seen": 2488074240
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9695073021986845e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.3982581331892603,
+      "tokens_seen": 2488205312
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9687048627828605e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.398243960780599,
+      "tokens_seen": 2488336384
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9679024233670365e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.3982297893274582,
+      "tokens_seen": 2488467456
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.967099983951212e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3982156188297226,
+      "tokens_seen": 2488598528
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.966297544535388e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3982014492872774,
+      "tokens_seen": 2488729600
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 835752,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2621967792510986,
+      "objective/train/theoretical_loss": 3.398187280700008,
+      "objective/train/tokens_used": 859319776,
+      "theoretical_loss": 3.398187280700008,
+      "tokens_seen": 2488860672
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.965495105119564e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.398187280700008,
+      "tokens_seen": 2488860672
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.96469266570374e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.398173113067801,
+      "tokens_seen": 2488991744
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.963890226287916e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3981589463905393,
+      "tokens_seen": 2489122816
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.963087786872092e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.39814478066811,
+      "tokens_seen": 2489253888
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.962285347456268e-05,
+      "loss": 1.1819,
+      "theoretical_loss": 3.3981306159003983,
+      "tokens_seen": 2489384960
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.961482908040443e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.398116452087289,
+      "tokens_seen": 2489516032
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.960680468624619e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3981022892286674,
+      "tokens_seen": 2489647104
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.959878029208795e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.39808812732442,
+      "tokens_seen": 2489778176
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.959075589792971e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.3980739663744313,
+      "tokens_seen": 2489909248
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.958273150377147e-05,
+      "loss": 1.1695,
+      "theoretical_loss": 3.398059806378587,
+      "tokens_seen": 2490040320
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.957470710961323e-05,
+      "loss": 1.2837,
+      "theoretical_loss": 3.398045647336773,
+      "tokens_seen": 2490171392
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.956668271545499e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3980314892488743,
+      "tokens_seen": 2490302464
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.955865832129675e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.3980173321147764,
+      "tokens_seen": 2490433536
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.95506339271385e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.398003175934365,
+      "tokens_seen": 2490564608
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.954260953298026e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.3979890207075263,
+      "tokens_seen": 2490695680
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.953458513882202e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.3979748664341445,
+      "tokens_seen": 2490826752
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.952656074466378e-05,
+      "loss": 1.288,
+      "theoretical_loss": 3.3979607131141067,
+      "tokens_seen": 2490957824
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.951853635050554e-05,
+      "loss": 1.1798,
+      "theoretical_loss": 3.3979465607472976,
+      "tokens_seen": 2491088896
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.95105119563473e-05,
+      "loss": 1.2693,
+      "theoretical_loss": 3.397932409333603,
+      "tokens_seen": 2491219968
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.950248756218906e-05,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3979182588729087,
+      "tokens_seen": 2491351040
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9494463168030815e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.3979041093651006,
+      "tokens_seen": 2491482112
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9486438773872575e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.3978899608100637,
+      "tokens_seen": 2491613184
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9478414379714335e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3978758132076847,
+      "tokens_seen": 2491744256
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9470389985556095e-05,
+      "loss": 1.3364,
+      "theoretical_loss": 3.3978616665578487,
+      "tokens_seen": 2491875328
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9462365591397855e-05,
+      "loss": 1.3061,
+      "theoretical_loss": 3.3978475208604415,
+      "tokens_seen": 2492006400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9454341197239615e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.397833376115349,
+      "tokens_seen": 2492137472
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9446316803081375e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.397819232322457,
+      "tokens_seen": 2492268544
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.943829240892313e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.3978050894816514,
+      "tokens_seen": 2492399616
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.943026801476489e-05,
+      "loss": 1.1903,
+      "theoretical_loss": 3.397790947592818,
+      "tokens_seen": 2492530688
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.942224362060665e-05,
+      "loss": 1.2226,
+      "theoretical_loss": 3.3977768066558425,
+      "tokens_seen": 2492661760
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.941421922644841e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3977626666706113,
+      "tokens_seen": 2492792832
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.940619483229017e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3977485276370096,
+      "tokens_seen": 2492923904
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.939817043813193e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3977343895549237,
+      "tokens_seen": 2493054976
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.939014604397368e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3977202524242394,
+      "tokens_seen": 2493186048
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.938212164981544e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.397706116244843,
+      "tokens_seen": 2493317120
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.93740972556572e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.39769198101662,
+      "tokens_seen": 2493448192
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.936607286149896e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.397677846739457,
+      "tokens_seen": 2493579264
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.935804846734072e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3976637134132393,
+      "tokens_seen": 2493710336
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.935002407318248e-05,
+      "loss": 1.284,
+      "theoretical_loss": 3.3976495810378537,
+      "tokens_seen": 2493841408
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.934199967902423e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.3976354496131855,
+      "tokens_seen": 2493972480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.933397528486599e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3976213191391214,
+      "tokens_seen": 2494103552
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.932595089070775e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.3976071896155475,
+      "tokens_seen": 2494234624
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.931792649654951e-05,
+      "loss": 1.3168,
+      "theoretical_loss": 3.3975930610423495,
+      "tokens_seen": 2494365696
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.930990210239127e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.3975789334194135,
+      "tokens_seen": 2494496768
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.930187770823303e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.397564806746626,
+      "tokens_seen": 2494627840
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9293853314074786e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3975506810238736,
+      "tokens_seen": 2494758912
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9285828919916546e-05,
+      "loss": 1.1765,
+      "theoretical_loss": 3.3975365562510413,
+      "tokens_seen": 2494889984
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9277804525758306e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3975224324280164,
+      "tokens_seen": 2495021056
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9269780131600066e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.3975083095546843,
+      "tokens_seen": 2495152128
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9261755737441826e-05,
+      "loss": 1.1774,
+      "theoretical_loss": 3.397494187630932,
+      "tokens_seen": 2495283200
+    },
+    {
+      "epoch": 0.51,
+      "objective/train/docs_used": 837794,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2158561944961548,
+      "objective/train/theoretical_loss": 3.397480066656646,
+      "objective/train/tokens_used": 865873376,
+      "theoretical_loss": 3.397480066656646,
+      "tokens_seen": 2495414272
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9253731343283586e-05,
+      "loss": 1.1805,
+      "theoretical_loss": 3.397480066656646,
+      "tokens_seen": 2495414272
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.924570694912534e-05,
+      "loss": 1.296,
+      "theoretical_loss": 3.397465946631711,
+      "tokens_seen": 2495545344
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.92376825549671e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.397451827556015,
+      "tokens_seen": 2495676416
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.922965816080886e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.397437709429444,
+      "tokens_seen": 2495807488
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.922163376665062e-05,
+      "loss": 1.3259,
+      "theoretical_loss": 3.397423592251884,
+      "tokens_seen": 2495938560
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.921360937249238e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.397409476023221,
+      "tokens_seen": 2496069632
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.920558497833414e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3973953607433423,
+      "tokens_seen": 2496200704
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.919756058417589e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.3973812464121336,
+      "tokens_seen": 2496331776
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.918953619001765e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.3973671330294817,
+      "tokens_seen": 2496462848
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.918151179585941e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.397353020595273,
+      "tokens_seen": 2496593920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.917348740170117e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3973389091093935,
+      "tokens_seen": 2496724992
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.916546300754293e-05,
+      "loss": 1.3017,
+      "theoretical_loss": 3.3973247985717308,
+      "tokens_seen": 2496856064
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.915743861338469e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3973106889821705,
+      "tokens_seen": 2496987136
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.914941421922645e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3972965803405994,
+      "tokens_seen": 2497118208
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9141389825068204e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.397282472646904,
+      "tokens_seen": 2497249280
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9133365430909964e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.3972683659009713,
+      "tokens_seen": 2497380352
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9125341036751724e-05,
+      "loss": 1.3305,
+      "theoretical_loss": 3.3972542601026867,
+      "tokens_seen": 2497511424
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9117316642593484e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.397240155251938,
+      "tokens_seen": 2497642496
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9109292248435244e-05,
+      "loss": 1.2885,
+      "theoretical_loss": 3.3972260513486114,
+      "tokens_seen": 2497773568
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9101267854277003e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.3972119483925938,
+      "tokens_seen": 2497904640
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.909324346011876e-05,
+      "loss": 1.1795,
+      "theoretical_loss": 3.3971978463837713,
+      "tokens_seen": 2498035712
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.908521906596052e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.3971837453220317,
+      "tokens_seen": 2498166784
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9077194671802276e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.39716964520726,
+      "tokens_seen": 2498297856
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9069170277644036e-05,
+      "loss": 1.322,
+      "theoretical_loss": 3.3971555460393446,
+      "tokens_seen": 2498428928
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9061145883485796e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.3971414478181714,
+      "tokens_seen": 2498560000
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9053121489327556e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3971273505436272,
+      "tokens_seen": 2498691072
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9045097095169316e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.397113254215599,
+      "tokens_seen": 2498822144
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9037072701011076e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.397099158833973,
+      "tokens_seen": 2498953216
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.9029048306852836e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3970850643986368,
+      "tokens_seen": 2499084288
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.902102391269459e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.397070970909477,
+      "tokens_seen": 2499215360
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.901299951853635e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3970568783663806,
+      "tokens_seen": 2499346432
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.900497512437811e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.397042786769234,
+      "tokens_seen": 2499477504
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 4.899695073021987e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.397028696117925,
+      "tokens_seen": 2499608576
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.898892633606163e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3970146064123394,
+      "tokens_seen": 2499739648
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.898090194190339e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.3970005176523648,
+      "tokens_seen": 2499870720
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.897287754774515e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.396986429837888,
+      "tokens_seen": 2500001792
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.896485315358691e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.396972342968796,
+      "tokens_seen": 2500132864
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.895682875942866e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.396958257044976,
+      "tokens_seen": 2500263936
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.894880436527042e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.3969441720663145,
+      "tokens_seen": 2500395008
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.894077997111218e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.3969300880326996,
+      "tokens_seen": 2500526080
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.893275557695394e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.396916004944017,
+      "tokens_seen": 2500657152
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.89247311827957e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3969019228001547,
+      "tokens_seen": 2500788224
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.891670678863746e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.396887841600999,
+      "tokens_seen": 2500919296
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.890868239447922e-05,
+      "loss": 1.311,
+      "theoretical_loss": 3.3968737613464386,
+      "tokens_seen": 2501050368
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8900658000320974e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.396859682036359,
+      "tokens_seen": 2501181440
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8892633606162734e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3968456036706476,
+      "tokens_seen": 2501312512
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8884609212004494e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3968315262491924,
+      "tokens_seen": 2501443584
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8876584817846254e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3968174497718797,
+      "tokens_seen": 2501574656
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8868560423688014e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.3968033742385972,
+      "tokens_seen": 2501705728
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8860536029529774e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.396789299649232,
+      "tokens_seen": 2501836800
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 839945,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2112829685211182,
+      "objective/train/theoretical_loss": 3.3967752260036717,
+      "objective/train/tokens_used": 872426976,
+      "theoretical_loss": 3.3967752260036717,
+      "tokens_seen": 2501967872
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8852511635371533e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3967752260036717,
+      "tokens_seen": 2501967872
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8844487241213287e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.3967611533018034,
+      "tokens_seen": 2502098944
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8836462847055046e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3967470815435137,
+      "tokens_seen": 2502230016
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8828438452896806e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.396733010728691,
+      "tokens_seen": 2502361088
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8820414058738566e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3967189408572223,
+      "tokens_seen": 2502492160
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8812389664580326e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.396704871928994,
+      "tokens_seen": 2502623232
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8804365270422086e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3966908039438946,
+      "tokens_seen": 2502754304
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8796340876263846e-05,
+      "loss": 1.1981,
+      "theoretical_loss": 3.396676736901811,
+      "tokens_seen": 2502885376
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8788316482105606e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.3966626708026313,
+      "tokens_seen": 2503016448
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.878029208794736e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.3966486056462415,
+      "tokens_seen": 2503147520
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.877226769378912e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.3966345414325305,
+      "tokens_seen": 2503278592
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.876424329963088e-05,
+      "loss": 1.3281,
+      "theoretical_loss": 3.3966204781613847,
+      "tokens_seen": 2503409664
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.875621890547264e-05,
+      "loss": 1.2919,
+      "theoretical_loss": 3.3966064158326925,
+      "tokens_seen": 2503540736
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.87481945113144e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.3965923544463408,
+      "tokens_seen": 2503671808
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.874017011715616e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.396578294002217,
+      "tokens_seen": 2503802880
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.873214572299792e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.396564234500209,
+      "tokens_seen": 2503933952
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.872412132883967e-05,
+      "loss": 1.1627,
+      "theoretical_loss": 3.3965501759402046,
+      "tokens_seen": 2504065024
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.871609693468143e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.396536118322091,
+      "tokens_seen": 2504196096
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.870807254052319e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.3965220616457557,
+      "tokens_seen": 2504327168
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.870004814636495e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3965080059110866,
+      "tokens_seen": 2504458240
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.869202375220671e-05,
+      "loss": 1.17,
+      "theoretical_loss": 3.396493951117971,
+      "tokens_seen": 2504589312
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.868399935804847e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.396479897266297,
+      "tokens_seen": 2504720384
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.867597496389023e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.396465844355952,
+      "tokens_seen": 2504851456
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8667950569731984e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.3964517923868236,
+      "tokens_seen": 2504982528
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8659926175573744e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.3964377413588,
+      "tokens_seen": 2505113600
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8651901781415504e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.3964236912717682,
+      "tokens_seen": 2505244672
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8643877387257264e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.396409642125617,
+      "tokens_seen": 2505375744
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8635852993099024e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.396395593920233,
+      "tokens_seen": 2505506816
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8627828598940784e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.3963815466555047,
+      "tokens_seen": 2505637888
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8619804204782544e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.3963675003313196,
+      "tokens_seen": 2505768960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8611779810624304e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.396353454947566,
+      "tokens_seen": 2505900032
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.860375541646606e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3963394105041314,
+      "tokens_seen": 2506031104
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8595731022307817e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.3963253670009035,
+      "tokens_seen": 2506162176
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8587706628149576e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.3963113244377703,
+      "tokens_seen": 2506293248
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8579682233991336e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.39629728281462,
+      "tokens_seen": 2506424320
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8571657839833096e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.3962832421313403,
+      "tokens_seen": 2506555392
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8563633445674856e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.396269202387819,
+      "tokens_seen": 2506686464
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8555609051516616e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.3962551635839446,
+      "tokens_seen": 2506817536
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.854758465735837e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.3962411257196043,
+      "tokens_seen": 2506948608
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.853956026320013e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.396227088794687,
+      "tokens_seen": 2507079680
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.853153586904189e-05,
+      "loss": 1.3255,
+      "theoretical_loss": 3.3962130528090797,
+      "tokens_seen": 2507210752
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.852351147488365e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.3961990177626715,
+      "tokens_seen": 2507341824
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.851548708072541e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.39618498365535,
+      "tokens_seen": 2507472896
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.850746268656717e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3961709504870026,
+      "tokens_seen": 2507603968
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.849943829240893e-05,
+      "loss": 1.224,
+      "theoretical_loss": 3.396156918257519,
+      "tokens_seen": 2507735040
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.849141389825069e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.3961428869667856,
+      "tokens_seen": 2507866112
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.848338950409244e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.3961288566146917,
+      "tokens_seen": 2507997184
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.84753651099342e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.396114827201125,
+      "tokens_seen": 2508128256
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.846734071577596e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.3961007987259735,
+      "tokens_seen": 2508259328
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.845931632161772e-05,
+      "loss": 1.3025,
+      "theoretical_loss": 3.396086771189126,
+      "tokens_seen": 2508390400
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 842199,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2736304998397827,
+      "objective/train/theoretical_loss": 3.39607274459047,
+      "objective/train/tokens_used": 878980576,
+      "theoretical_loss": 3.39607274459047,
+      "tokens_seen": 2508521472
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.845129192745948e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.39607274459047,
+      "tokens_seen": 2508521472
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.844326753330124e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.3960587189298943,
+      "tokens_seen": 2508652544
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8435243139143e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.396044694207287,
+      "tokens_seen": 2508783616
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8427218744984754e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.3960306704225363,
+      "tokens_seen": 2508914688
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8419194350826514e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3960166475755305,
+      "tokens_seen": 2509045760
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8411169956668274e-05,
+      "loss": 1.2226,
+      "theoretical_loss": 3.3960026256661577,
+      "tokens_seen": 2509176832
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8403145562510034e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3959886046943066,
+      "tokens_seen": 2509307904
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8395121168351794e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3959745846598657,
+      "tokens_seen": 2509438976
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8387096774193554e-05,
+      "loss": 1.2067,
+      "theoretical_loss": 3.395960565562723,
+      "tokens_seen": 2509570048
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8379072380035314e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.3959465474027666,
+      "tokens_seen": 2509701120
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.837104798587707e-05,
+      "loss": 1.291,
+      "theoretical_loss": 3.3959325301798855,
+      "tokens_seen": 2509832192
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.836302359171883e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.3959185138939683,
+      "tokens_seen": 2509963264
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.835499919756059e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.3959044985449025,
+      "tokens_seen": 2510094336
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8346974803402347e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.395890484132577,
+      "tokens_seen": 2510225408
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8338950409244106e-05,
+      "loss": 1.2989,
+      "theoretical_loss": 3.395876470656881,
+      "tokens_seen": 2510356480
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8330926015085866e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3958624581177017,
+      "tokens_seen": 2510487552
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8322901620927626e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.395848446514929,
+      "tokens_seen": 2510618624
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8314877226769386e-05,
+      "loss": 1.3207,
+      "theoretical_loss": 3.3958344358484505,
+      "tokens_seen": 2510749696
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.830685283261114e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.3958204261181546,
+      "tokens_seen": 2510880768
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.82988284384529e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.395806417323931,
+      "tokens_seen": 2511011840
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.829080404429466e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.3957924094656673,
+      "tokens_seen": 2511142912
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.828277965013642e-05,
+      "loss": 1.3828,
+      "theoretical_loss": 3.3957784025432525,
+      "tokens_seen": 2511273984
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.827475525597818e-05,
+      "loss": 1.3158,
+      "theoretical_loss": 3.395764396556575,
+      "tokens_seen": 2511405056
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.826673086181994e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.3957503915055236,
+      "tokens_seen": 2511536128
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.82587064676617e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.395736387389987,
+      "tokens_seen": 2511667200
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.825068207350345e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3957223842098543,
+      "tokens_seen": 2511798272
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.824265767934521e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.3957083819650133,
+      "tokens_seen": 2511929344
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.823463328518697e-05,
+      "loss": 1.2171,
+      "theoretical_loss": 3.3956943806553532,
+      "tokens_seen": 2512060416
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.822660889102873e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.395680380280763,
+      "tokens_seen": 2512191488
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.821858449687049e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.3956663808411314,
+      "tokens_seen": 2512322560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.821056010271225e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3956523823363467,
+      "tokens_seen": 2512453632
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.820253570855401e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.3956383847662983,
+      "tokens_seen": 2512584704
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8194511314395764e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.3956243881308747,
+      "tokens_seen": 2512715776
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8186486920237524e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.3956103924299645,
+      "tokens_seen": 2512846848
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8178462526079284e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.395596397663457,
+      "tokens_seen": 2512977920
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8170438131921044e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3955824038312414,
+      "tokens_seen": 2513108992
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8162413737762804e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.395568410933206,
+      "tokens_seen": 2513240064
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8154389343604564e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.39555441896924,
+      "tokens_seen": 2513371136
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8146364949446324e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.395540427939232,
+      "tokens_seen": 2513502208
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8138340555288084e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.395526437843071,
+      "tokens_seen": 2513633280
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.813031616112984e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3955124486806465,
+      "tokens_seen": 2513764352
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.81222917669716e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.3954984604518472,
+      "tokens_seen": 2513895424
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.811426737281336e-05,
+      "loss": 1.313,
+      "theoretical_loss": 3.3954844731565617,
+      "tokens_seen": 2514026496
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8106242978655117e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.3954704867946797,
+      "tokens_seen": 2514157568
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8098218584496877e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.3954565013660902,
+      "tokens_seen": 2514288640
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8090194190338636e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.395442516870682,
+      "tokens_seen": 2514419712
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.8082169796180396e-05,
+      "loss": 1.2959,
+      "theoretical_loss": 3.395428533308344,
+      "tokens_seen": 2514550784
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.807414540202215e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3954145506789657,
+      "tokens_seen": 2514681856
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.806612100786391e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.395400568982436,
+      "tokens_seen": 2514812928
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.805809661370567e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3953865882186443,
+      "tokens_seen": 2514944000
+    },
+    {
+      "epoch": 0.52,
+      "objective/train/docs_used": 844467,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.277206540107727,
+      "objective/train/theoretical_loss": 3.395372608387479,
+      "objective/train/tokens_used": 885534176,
+      "theoretical_loss": 3.395372608387479,
+      "tokens_seen": 2515075072
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.805007221954743e-05,
+      "loss": 1.3045,
+      "theoretical_loss": 3.395372608387479,
+      "tokens_seen": 2515075072
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.804204782538919e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.3953586294888307,
+      "tokens_seen": 2515206144
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.803402343123095e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3953446515225876,
+      "tokens_seen": 2515337216
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.80259990370727e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.3953306744886387,
+      "tokens_seen": 2515468288
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.801797464291446e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3953166983868743,
+      "tokens_seen": 2515599360
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.800995024875622e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.3953027232171826,
+      "tokens_seen": 2515730432
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.800192585459798e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3952887489794534,
+      "tokens_seen": 2515861504
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.799390146043974e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.395274775673576,
+      "tokens_seen": 2515992576
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.79858770662815e-05,
+      "loss": 1.303,
+      "theoretical_loss": 3.39526080329944,
+      "tokens_seen": 2516123648
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7977852672123255e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.3952468318569338,
+      "tokens_seen": 2516254720
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7969828277965015e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.3952328613459475,
+      "tokens_seen": 2516385792
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7961803883806775e-05,
+      "loss": 1.3139,
+      "theoretical_loss": 3.395218891766371,
+      "tokens_seen": 2516516864
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7953779489648534e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.395204923118092,
+      "tokens_seen": 2516647936
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7945755095490294e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.3951909554010014,
+      "tokens_seen": 2516779008
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7937730701332054e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3951769886149883,
+      "tokens_seen": 2516910080
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.792970630717381e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.3951630227599416,
+      "tokens_seen": 2517041152
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.792168191301557e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.3951490578357513,
+      "tokens_seen": 2517172224
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.791365751885733e-05,
+      "loss": 1.3059,
+      "theoretical_loss": 3.395135093842307,
+      "tokens_seen": 2517303296
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.790563312469909e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.395121130779498,
+      "tokens_seen": 2517434368
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.789760873054085e-05,
+      "loss": 1.2604,
+      "theoretical_loss": 3.3951071686472134,
+      "tokens_seen": 2517565440
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.78895843363826e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.3950932074453433,
+      "tokens_seen": 2517696512
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.788155994222436e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3950792471737774,
+      "tokens_seen": 2517827584
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.787353554806612e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.395065287832405,
+      "tokens_seen": 2517958656
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.786551115390788e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3950513294211158,
+      "tokens_seen": 2518089728
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.785748675974964e-05,
+      "loss": 1.3382,
+      "theoretical_loss": 3.395037371939799,
+      "tokens_seen": 2518220800
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.78494623655914e-05,
+      "loss": 1.1842,
+      "theoretical_loss": 3.3950234153883443,
+      "tokens_seen": 2518351872
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.784143797143316e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3950094597666425,
+      "tokens_seen": 2518482944
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.783341357727491e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.394995505074582,
+      "tokens_seen": 2518614016
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.782538918311667e-05,
+      "loss": 1.1421,
+      "theoretical_loss": 3.3949815513120525,
+      "tokens_seen": 2518745088
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.781736478895843e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.394967598478945,
+      "tokens_seen": 2518876160
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.780934039480019e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3949536465751478,
+      "tokens_seen": 2519007232
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.780131600064195e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.3949396956005513,
+      "tokens_seen": 2519138304
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.779329160648371e-05,
+      "loss": 1.314,
+      "theoretical_loss": 3.394925745555045,
+      "tokens_seen": 2519269376
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.778526721232547e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.394911796438519,
+      "tokens_seen": 2519400448
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7777242818167225e-05,
+      "loss": 1.1482,
+      "theoretical_loss": 3.394897848250863,
+      "tokens_seen": 2519531520
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7769218424008985e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.3948839009919674,
+      "tokens_seen": 2519662592
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7761194029850745e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.3948699546617207,
+      "tokens_seen": 2519793664
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7753169635692505e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.394856009260014,
+      "tokens_seen": 2519924736
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7745145241534265e-05,
+      "loss": 1.2902,
+      "theoretical_loss": 3.3948420647867366,
+      "tokens_seen": 2520055808
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7737120847376025e-05,
+      "loss": 1.1593,
+      "theoretical_loss": 3.394828121241779,
+      "tokens_seen": 2520186880
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7729096453217785e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.3948141786250297,
+      "tokens_seen": 2520317952
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.772107205905954e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.39480023693638,
+      "tokens_seen": 2520449024
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.77130476649013e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.39478629617572,
+      "tokens_seen": 2520580096
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.770502327074306e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.3947723563429384,
+      "tokens_seen": 2520711168
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.769699887658482e-05,
+      "loss": 1.3074,
+      "theoretical_loss": 3.394758417437926,
+      "tokens_seen": 2520842240
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.768897448242658e-05,
+      "loss": 1.3299,
+      "theoretical_loss": 3.394744479460573,
+      "tokens_seen": 2520973312
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.768095008826834e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.394730542410769,
+      "tokens_seen": 2521104384
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.76729256941101e-05,
+      "loss": 1.2819,
+      "theoretical_loss": 3.3947166062884047,
+      "tokens_seen": 2521235456
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.766490129995186e-05,
+      "loss": 1.3065,
+      "theoretical_loss": 3.3947026710933694,
+      "tokens_seen": 2521366528
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.765687690579361e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.3946887368255534,
+      "tokens_seen": 2521497600
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 846407,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2216236591339111,
+      "objective/train/theoretical_loss": 3.3946748034848473,
+      "objective/train/tokens_used": 892087776,
+      "theoretical_loss": 3.3946748034848473,
+      "tokens_seen": 2521628672
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.764885251163537e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.3946748034848473,
+      "tokens_seen": 2521628672
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.764082811747713e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.3946608710711406,
+      "tokens_seen": 2521759744
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.763280372331889e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3946469395843235,
+      "tokens_seen": 2521890816
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.762477932916065e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.394633009024287,
+      "tokens_seen": 2522021888
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.761675493500241e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.39461907939092,
+      "tokens_seen": 2522152960
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.760873054084417e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3946051506841135,
+      "tokens_seen": 2522284032
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.760070614668592e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.394591222903758,
+      "tokens_seen": 2522415104
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.759268175252768e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.394577296049743,
+      "tokens_seen": 2522546176
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.758465735836944e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.3945633701219586,
+      "tokens_seen": 2522677248
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.75766329642112e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.3945494451202967,
+      "tokens_seen": 2522808320
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.756860857005296e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.3945355210446455,
+      "tokens_seen": 2522939392
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.756058417589472e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3945215978948964,
+      "tokens_seen": 2523070464
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.755255978173648e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.39450767567094,
+      "tokens_seen": 2523201536
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.754453538757824e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.3944937543726663,
+      "tokens_seen": 2523332608
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7536510993419995e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.394479833999965,
+      "tokens_seen": 2523463680
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7528486599261755e-05,
+      "loss": 1.147,
+      "theoretical_loss": 3.394465914552728,
+      "tokens_seen": 2523594752
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7520462205103515e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.394451996030844,
+      "tokens_seen": 2523725824
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7512437810945275e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.394438078434205,
+      "tokens_seen": 2523856896
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7504413416787035e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3944241617627005,
+      "tokens_seen": 2523987968
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7496389022628795e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.394410246016221,
+      "tokens_seen": 2524119040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7488364628470555e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.394396331194657,
+      "tokens_seen": 2524250112
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.748034023431231e-05,
+      "loss": 1.2871,
+      "theoretical_loss": 3.3943824172978996,
+      "tokens_seen": 2524381184
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.747231584015407e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.394368504325839,
+      "tokens_seen": 2524512256
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.746429144599583e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.394354592278365,
+      "tokens_seen": 2524643328
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.745626705183759e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.394340681155369,
+      "tokens_seen": 2524774400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.744824265767935e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.394326770956742,
+      "tokens_seen": 2524905472
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.744021826352111e-05,
+      "loss": 1.3389,
+      "theoretical_loss": 3.3943128616823737,
+      "tokens_seen": 2525036544
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.743219386936287e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.3942989533321546,
+      "tokens_seen": 2525167616
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.742416947520462e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.394285045905976,
+      "tokens_seen": 2525298688
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.741614508104638e-05,
+      "loss": 1.1671,
+      "theoretical_loss": 3.394271139403728,
+      "tokens_seen": 2525429760
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.740812068688814e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.394257233825302,
+      "tokens_seen": 2525560832
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.74000962927299e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.394243329170588,
+      "tokens_seen": 2525691904
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.739207189857166e-05,
+      "loss": 1.3027,
+      "theoretical_loss": 3.3942294254394767,
+      "tokens_seen": 2525822976
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.738404750441342e-05,
+      "loss": 1.337,
+      "theoretical_loss": 3.3942155226318595,
+      "tokens_seen": 2525954048
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.737602311025518e-05,
+      "loss": 1.3121,
+      "theoretical_loss": 3.3942016207476264,
+      "tokens_seen": 2526085120
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.736799871609694e-05,
+      "loss": 1.2026,
+      "theoretical_loss": 3.3941877197866686,
+      "tokens_seen": 2526216192
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.735997432193869e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3941738197488767,
+      "tokens_seen": 2526347264
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.735194992778045e-05,
+      "loss": 1.2802,
+      "theoretical_loss": 3.3941599206341415,
+      "tokens_seen": 2526478336
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.734392553362221e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.3941460224423543,
+      "tokens_seen": 2526609408
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.733590113946397e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.394132125173405,
+      "tokens_seen": 2526740480
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.732787674530573e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.3941182288271854,
+      "tokens_seen": 2526871552
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.731985235114749e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.3941043334035856,
+      "tokens_seen": 2527002624
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.731182795698925e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3940904389024973,
+      "tokens_seen": 2527133696
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7303803562831006e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.3940765453238106,
+      "tokens_seen": 2527264768
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7295779168672765e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.394062652667417,
+      "tokens_seen": 2527395840
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7287754774514525e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.394048760933207,
+      "tokens_seen": 2527526912
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7279730380356285e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.394034870121072,
+      "tokens_seen": 2527657984
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7271705986198045e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.394020980230903,
+      "tokens_seen": 2527789056
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7263681592039805e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.3940070912625906,
+      "tokens_seen": 2527920128
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7255657197881565e-05,
+      "loss": 1.1186,
+      "theoretical_loss": 3.393993203216026,
+      "tokens_seen": 2528051200
+    },
+    {
+      "epoch": 0.53,
+      "objective/train/docs_used": 848730,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2010843753814697,
+      "objective/train/theoretical_loss": 3.3939793160911,
+      "objective/train/tokens_used": 898641376,
+      "theoretical_loss": 3.3939793160911,
+      "tokens_seen": 2528182272
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.724763280372332e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.3939793160911,
+      "tokens_seen": 2528182272
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.723960840956508e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.393965429887704,
+      "tokens_seen": 2528313344
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.723158401540684e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.3939515446057293,
+      "tokens_seen": 2528444416
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.72235596212486e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.393937660245067,
+      "tokens_seen": 2528575488
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.721553522709036e-05,
+      "loss": 1.2837,
+      "theoretical_loss": 3.3939237768056074,
+      "tokens_seen": 2528706560
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.720751083293212e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3939098942872423,
+      "tokens_seen": 2528837632
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.719948643877388e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.3938960126898623,
+      "tokens_seen": 2528968704
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.719146204461564e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.39388213201336,
+      "tokens_seen": 2529099776
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.718343765045739e-05,
+      "loss": 1.3374,
+      "theoretical_loss": 3.393868252257625,
+      "tokens_seen": 2529230848
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.717541325629915e-05,
+      "loss": 1.3035,
+      "theoretical_loss": 3.393854373422549,
+      "tokens_seen": 2529361920
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.716738886214091e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.393840495508023,
+      "tokens_seen": 2529492992
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.715936446798267e-05,
+      "loss": 1.3005,
+      "theoretical_loss": 3.3938266185139394,
+      "tokens_seen": 2529624064
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.715134007382443e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.393812742440188,
+      "tokens_seen": 2529755136
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.714331567966619e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3937988672866606,
+      "tokens_seen": 2529886208
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.713529128550795e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.393784993053249,
+      "tokens_seen": 2530017280
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.71272668913497e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3937711197398444,
+      "tokens_seen": 2530148352
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.711924249719146e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.3937572473463375,
+      "tokens_seen": 2530279424
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.711121810303322e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.39374337587262,
+      "tokens_seen": 2530410496
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.710319370887498e-05,
+      "loss": 1.2859,
+      "theoretical_loss": 3.3937295053185834,
+      "tokens_seen": 2530541568
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.709516931471674e-05,
+      "loss": 1.3189,
+      "theoretical_loss": 3.393715635684119,
+      "tokens_seen": 2530672640
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.70871449205585e-05,
+      "loss": 1.3012,
+      "theoretical_loss": 3.393701766969118,
+      "tokens_seen": 2530803712
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.707912052640026e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3936878991734725,
+      "tokens_seen": 2530934784
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.707109613224202e-05,
+      "loss": 1.3355,
+      "theoretical_loss": 3.393674032297073,
+      "tokens_seen": 2531065856
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7063071738083776e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.393660166339812,
+      "tokens_seen": 2531196928
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7055047343925536e-05,
+      "loss": 1.3433,
+      "theoretical_loss": 3.3936463013015796,
+      "tokens_seen": 2531328000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7047022949767295e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.393632437182269,
+      "tokens_seen": 2531459072
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7038998555609055e-05,
+      "loss": 1.1783,
+      "theoretical_loss": 3.3936185739817706,
+      "tokens_seen": 2531590144
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7030974161450815e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.393604711699976,
+      "tokens_seen": 2531721216
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7022949767292575e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.393590850336777,
+      "tokens_seen": 2531852288
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.7014925373134335e-05,
+      "loss": 1.3224,
+      "theoretical_loss": 3.393576989892065,
+      "tokens_seen": 2531983360
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.700690097897609e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.3935631303657323,
+      "tokens_seen": 2532114432
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.699887658481785e-05,
+      "loss": 1.1687,
+      "theoretical_loss": 3.39354927175767,
+      "tokens_seen": 2532245504
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.699085219065961e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3935354140677694,
+      "tokens_seen": 2532376576
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.698282779650137e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.393521557295922,
+      "tokens_seen": 2532507648
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.697480340234313e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.3935077014420205,
+      "tokens_seen": 2532638720
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.696677900818489e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.393493846505956,
+      "tokens_seen": 2532769792
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.695875461402665e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.39347999248762,
+      "tokens_seen": 2532900864
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.69507302198684e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3934661393869043,
+      "tokens_seen": 2533031936
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.694270582571016e-05,
+      "loss": 1.2927,
+      "theoretical_loss": 3.393452287203701,
+      "tokens_seen": 2533163008
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.693468143155192e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.393438435937902,
+      "tokens_seen": 2533294080
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.692665703739368e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.393424585589398,
+      "tokens_seen": 2533425152
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.691863264323544e-05,
+      "loss": 1.1948,
+      "theoretical_loss": 3.393410736158082,
+      "tokens_seen": 2533556224
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.69106082490772e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.393396887643845,
+      "tokens_seen": 2533687296
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.690258385491896e-05,
+      "loss": 1.2892,
+      "theoretical_loss": 3.3933830400465794,
+      "tokens_seen": 2533818368
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.689455946076072e-05,
+      "loss": 1.3432,
+      "theoretical_loss": 3.393369193366177,
+      "tokens_seen": 2533949440
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.688653506660247e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3933553476025295,
+      "tokens_seen": 2534080512
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.687851067244423e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.3933415027555287,
+      "tokens_seen": 2534211584
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.687048627828599e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.3933276588250667,
+      "tokens_seen": 2534342656
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.686246188412775e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.3933138158110348,
+      "tokens_seen": 2534473728
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.685443748996951e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.393299973713326,
+      "tokens_seen": 2534604800
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 850982,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2104291915893555,
+      "objective/train/theoretical_loss": 3.3932861325318315,
+      "objective/train/tokens_used": 905194976,
+      "theoretical_loss": 3.3932861325318315,
+      "tokens_seen": 2534735872
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.684641309581127e-05,
+      "loss": 1.1938,
+      "theoretical_loss": 3.3932861325318315,
+      "tokens_seen": 2534735872
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.683838870165303e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3932722922664436,
+      "tokens_seen": 2534866944
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6830364307494786e-05,
+      "loss": 1.3134,
+      "theoretical_loss": 3.3932584529170544,
+      "tokens_seen": 2534998016
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6822339913336546e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3932446144835557,
+      "tokens_seen": 2535129088
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6814315519178306e-05,
+      "loss": 1.2113,
+      "theoretical_loss": 3.393230776965839,
+      "tokens_seen": 2535260160
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6806291125020066e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.3932169403637977,
+      "tokens_seen": 2535391232
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6798266730861825e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.393203104677323,
+      "tokens_seen": 2535522304
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6790242336703585e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3931892699063066,
+      "tokens_seen": 2535653376
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6782217942545345e-05,
+      "loss": 1.3864,
+      "theoretical_loss": 3.3931754360506416,
+      "tokens_seen": 2535784448
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.67741935483871e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.3931616031102196,
+      "tokens_seen": 2535915520
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.676616915422886e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.3931477710849327,
+      "tokens_seen": 2536046592
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.675814476007062e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.393133939974673,
+      "tokens_seen": 2536177664
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.675012036591238e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.393120109779333,
+      "tokens_seen": 2536308736
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.674209597175414e-05,
+      "loss": 1.1845,
+      "theoretical_loss": 3.3931062804988046,
+      "tokens_seen": 2536439808
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.67340715775959e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.39309245213298,
+      "tokens_seen": 2536570880
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.672604718343766e-05,
+      "loss": 1.2993,
+      "theoretical_loss": 3.393078624681752,
+      "tokens_seen": 2536701952
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.671802278927941e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.3930647981450126,
+      "tokens_seen": 2536833024
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.670999839512117e-05,
+      "loss": 1.2752,
+      "theoretical_loss": 3.3930509725226536,
+      "tokens_seen": 2536964096
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.670197400096293e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.3930371478145673,
+      "tokens_seen": 2537095168
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.669394960680469e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.393023324020647,
+      "tokens_seen": 2537226240
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.668592521264645e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.393009501140784,
+      "tokens_seen": 2537357312
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.667790081848821e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.392995679174871,
+      "tokens_seen": 2537488384
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6669876424329964e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3929818581228006,
+      "tokens_seen": 2537619456
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6661852030171723e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.392968037984465,
+      "tokens_seen": 2537750528
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6653827636013483e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.392954218759756,
+      "tokens_seen": 2537881600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.664580324185524e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.392940400448567,
+      "tokens_seen": 2538012672
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6637778847697e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.39292658305079,
+      "tokens_seen": 2538143744
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.662975445353876e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.3929127665663175,
+      "tokens_seen": 2538274816
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6621730059380516e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.392898950995042,
+      "tokens_seen": 2538405888
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6613705665222276e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.392885136336856,
+      "tokens_seen": 2538536960
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6605681271064036e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.3928713225916516,
+      "tokens_seen": 2538668032
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6597656876905796e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.392857509759322,
+      "tokens_seen": 2538799104
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6589632482747556e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.3928436978397594,
+      "tokens_seen": 2538930176
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6581608088589316e-05,
+      "loss": 1.2989,
+      "theoretical_loss": 3.392829886832857,
+      "tokens_seen": 2539061248
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.657358369443107e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3928160767385056,
+      "tokens_seen": 2539192320
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.656555930027283e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.3928022675565996,
+      "tokens_seen": 2539323392
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.655753490611459e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.392788459287031,
+      "tokens_seen": 2539454464
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.654951051195635e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3927746519296926,
+      "tokens_seen": 2539585536
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.654148611779811e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3927608454844767,
+      "tokens_seen": 2539716608
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.653346172363987e-05,
+      "loss": 1.3294,
+      "theoretical_loss": 3.392747039951276,
+      "tokens_seen": 2539847680
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.652543732948162e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3927332353299837,
+      "tokens_seen": 2539978752
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.651741293532338e-05,
+      "loss": 1.3259,
+      "theoretical_loss": 3.3927194316204923,
+      "tokens_seen": 2540109824
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.650938854116514e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.392705628822694,
+      "tokens_seen": 2540240896
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.65013641470069e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3926918269364825,
+      "tokens_seen": 2540371968
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.649333975284866e-05,
+      "loss": 1.29,
+      "theoretical_loss": 3.3926780259617497,
+      "tokens_seen": 2540503040
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.648531535869042e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.392664225898389,
+      "tokens_seen": 2540634112
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6477290964532174e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3926504267462927,
+      "tokens_seen": 2540765184
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6469266570373934e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.3926366285053535,
+      "tokens_seen": 2540896256
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6461242176215694e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.392622831175465,
+      "tokens_seen": 2541027328
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6453217782057454e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.3926090347565196,
+      "tokens_seen": 2541158400
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 852916,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2142835855484009,
+      "objective/train/theoretical_loss": 3.3925952392484104,
+      "objective/train/tokens_used": 911748576,
+      "theoretical_loss": 3.3925952392484104,
+      "tokens_seen": 2541289472
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6445193387899214e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3925952392484104,
+      "tokens_seen": 2541289472
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6437168993740974e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.39258144465103,
+      "tokens_seen": 2541420544
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6429144599582734e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.3925676509642715,
+      "tokens_seen": 2541551616
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6421120205424494e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.3925538581880272,
+      "tokens_seen": 2541682688
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.641309581126625e-05,
+      "loss": 1.3169,
+      "theoretical_loss": 3.392540066322191,
+      "tokens_seen": 2541813760
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6405071417108007e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.392526275366656,
+      "tokens_seen": 2541944832
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6397047022949767e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.3925124853213138,
+      "tokens_seen": 2542075904
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6389022628791526e-05,
+      "loss": 1.2834,
+      "theoretical_loss": 3.3924986961860584,
+      "tokens_seen": 2542206976
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6380998234633286e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.392484907960783,
+      "tokens_seen": 2542338048
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6372973840475046e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.3924711206453804,
+      "tokens_seen": 2542469120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6364949446316806e-05,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3924573342397433,
+      "tokens_seen": 2542600192
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.635692505215856e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3924435487437647,
+      "tokens_seen": 2542731264
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.634890065800032e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.3924297641573387,
+      "tokens_seen": 2542862336
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.634087626384208e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.392415980480357,
+      "tokens_seen": 2542993408
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.633285186968384e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.3924021977127143,
+      "tokens_seen": 2543124480
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.63248274755256e-05,
+      "loss": 1.2844,
+      "theoretical_loss": 3.3923884158543025,
+      "tokens_seen": 2543255552
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.631680308136736e-05,
+      "loss": 1.3173,
+      "theoretical_loss": 3.3923746349050155,
+      "tokens_seen": 2543386624
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.630877868720912e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.3923608548647457,
+      "tokens_seen": 2543517696
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.630075429305087e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.3923470757333867,
+      "tokens_seen": 2543648768
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.629272989889263e-05,
+      "loss": 1.3233,
+      "theoretical_loss": 3.3923332975108322,
+      "tokens_seen": 2543779840
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.628470550473439e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.392319520196975,
+      "tokens_seen": 2543910912
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.627668111057615e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.392305743791708,
+      "tokens_seen": 2544041984
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.626865671641791e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.3922919682949253,
+      "tokens_seen": 2544173056
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.626063232225967e-05,
+      "loss": 1.3324,
+      "theoretical_loss": 3.3922781937065194,
+      "tokens_seen": 2544304128
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.625260792810143e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3922644200263843,
+      "tokens_seen": 2544435200
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.624458353394319e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3922506472544125,
+      "tokens_seen": 2544566272
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6236559139784944e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3922368753904983,
+      "tokens_seen": 2544697344
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6228534745626704e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3922231044345343,
+      "tokens_seen": 2544828416
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6220510351468464e-05,
+      "loss": 1.2817,
+      "theoretical_loss": 3.3922093343864144,
+      "tokens_seen": 2544959488
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6212485957310224e-05,
+      "loss": 1.1739,
+      "theoretical_loss": 3.3921955652460314,
+      "tokens_seen": 2545090560
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6204461563151984e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.3921817970132793,
+      "tokens_seen": 2545221632
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6196437168993744e-05,
+      "loss": 1.1743,
+      "theoretical_loss": 3.3921680296880514,
+      "tokens_seen": 2545352704
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6188412774835504e-05,
+      "loss": 1.3804,
+      "theoretical_loss": 3.3921542632702413,
+      "tokens_seen": 2545483776
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.618038838067726e-05,
+      "loss": 1.2808,
+      "theoretical_loss": 3.392140497759742,
+      "tokens_seen": 2545614848
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.617236398651902e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.392126733156447,
+      "tokens_seen": 2545745920
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.616433959236078e-05,
+      "loss": 1.2867,
+      "theoretical_loss": 3.3921129694602503,
+      "tokens_seen": 2545876992
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6156315198202537e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.3920992066710447,
+      "tokens_seen": 2546008064
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6148290804044296e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.3920854447887248,
+      "tokens_seen": 2546139136
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6140266409886056e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.3920716838131835,
+      "tokens_seen": 2546270208
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6132242015727816e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.3920579237443143,
+      "tokens_seen": 2546401280
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.612421762156957e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.392044164582011,
+      "tokens_seen": 2546532352
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.611619322741133e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.392030406326167,
+      "tokens_seen": 2546663424
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.610816883325309e-05,
+      "loss": 1.2888,
+      "theoretical_loss": 3.3920166489766763,
+      "tokens_seen": 2546794496
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.610014443909485e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.3920028925334322,
+      "tokens_seen": 2546925568
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.609212004493661e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3919891369963286,
+      "tokens_seen": 2547056640
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.608409565077837e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.391975382365259,
+      "tokens_seen": 2547187712
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.607607125662013e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.391961628640117,
+      "tokens_seen": 2547318784
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.606804686246189e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.3919478758207973,
+      "tokens_seen": 2547449856
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.606002246830364e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.391934123907192,
+      "tokens_seen": 2547580928
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.60519980741454e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3919203728991962,
+      "tokens_seen": 2547712000
+    },
+    {
+      "epoch": 0.54,
+      "objective/train/docs_used": 855143,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2466826438903809,
+      "objective/train/theoretical_loss": 3.391906622796703,
+      "objective/train/tokens_used": 918302176,
+      "theoretical_loss": 3.391906622796703,
+      "tokens_seen": 2547843072
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.604397367998716e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.391906622796703,
+      "tokens_seen": 2547843072
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.603594928582892e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3918928735996063,
+      "tokens_seen": 2547974144
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.602792489167068e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.3918791253078004,
+      "tokens_seen": 2548105216
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.601990049751244e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.3918653779211785,
+      "tokens_seen": 2548236288
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.60118761033542e-05,
+      "loss": 1.3215,
+      "theoretical_loss": 3.3918516314396348,
+      "tokens_seen": 2548367360
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.6003851709195954e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.391837885863063,
+      "tokens_seen": 2548498432
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.5995827315037714e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.391824141191357,
+      "tokens_seen": 2548629504
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.5987802920879474e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.3918103974244107,
+      "tokens_seen": 2548760576
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.5979778526721234e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.391796654562118,
+      "tokens_seen": 2548891648
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.5971754132562994e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.391782912604373,
+      "tokens_seen": 2549022720
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 4.5963729738404754e-05,
+      "loss": 1.1855,
+      "theoretical_loss": 3.39176917155107,
+      "tokens_seen": 2549153792
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5955705344246514e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.391755431402102,
+      "tokens_seen": 2549284864
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5947680950088274e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3917416921573635,
+      "tokens_seen": 2549415936
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.593965655593003e-05,
+      "loss": 1.269,
+      "theoretical_loss": 3.3917279538167486,
+      "tokens_seen": 2549547008
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.593163216177179e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.391714216380151,
+      "tokens_seen": 2549678080
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.592360776761355e-05,
+      "loss": 1.3543,
+      "theoretical_loss": 3.3917004798474655,
+      "tokens_seen": 2549809152
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.591558337345531e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.3916867442185854,
+      "tokens_seen": 2549940224
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5907558979297067e-05,
+      "loss": 1.3032,
+      "theoretical_loss": 3.391673009493405,
+      "tokens_seen": 2550071296
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5899534585138826e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.3916592756718185,
+      "tokens_seen": 2550202368
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5891510190980586e-05,
+      "loss": 1.3345,
+      "theoretical_loss": 3.39164554275372,
+      "tokens_seen": 2550333440
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.588348579682234e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.3916318107390038,
+      "tokens_seen": 2550464512
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.58754614026641e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3916180796275635,
+      "tokens_seen": 2550595584
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.586743700850586e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.3916043494192936,
+      "tokens_seen": 2550726656
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.585941261434762e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3915906201140884,
+      "tokens_seen": 2550857728
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.585138822018938e-05,
+      "loss": 1.3413,
+      "theoretical_loss": 3.391576891711842,
+      "tokens_seen": 2550988800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.584336382603114e-05,
+      "loss": 1.3249,
+      "theoretical_loss": 3.3915631642124486,
+      "tokens_seen": 2551119872
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.58353394318729e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.3915494376158026,
+      "tokens_seen": 2551250944
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.582731503771465e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.391535711921798,
+      "tokens_seen": 2551382016
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.581929064355641e-05,
+      "loss": 1.3207,
+      "theoretical_loss": 3.3915219871303295,
+      "tokens_seen": 2551513088
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.581126624939817e-05,
+      "loss": 1.3175,
+      "theoretical_loss": 3.3915082632412905,
+      "tokens_seen": 2551644160
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.580324185523993e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.3914945402545764,
+      "tokens_seen": 2551775232
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.579521746108169e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.3914808181700806,
+      "tokens_seen": 2551906304
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.578719306692345e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3914670969876983,
+      "tokens_seen": 2552037376
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.577916867276521e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.3914533767073234,
+      "tokens_seen": 2552168448
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.577114427860697e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.3914396573288506,
+      "tokens_seen": 2552299520
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5763119884448725e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.3914259388521737,
+      "tokens_seen": 2552430592
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5755095490290484e-05,
+      "loss": 1.1557,
+      "theoretical_loss": 3.3914122212771876,
+      "tokens_seen": 2552561664
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5747071096132244e-05,
+      "loss": 1.2978,
+      "theoretical_loss": 3.391398504603786,
+      "tokens_seen": 2552692736
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5739046701974004e-05,
+      "loss": 1.2872,
+      "theoretical_loss": 3.3913847888318647,
+      "tokens_seen": 2552823808
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5731022307815764e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.3913710739613174,
+      "tokens_seen": 2552954880
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5722997913657524e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.391357359992038,
+      "tokens_seen": 2553085952
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5714973519499284e-05,
+      "loss": 1.2986,
+      "theoretical_loss": 3.391343646923922,
+      "tokens_seen": 2553217024
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.570694912534104e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3913299347568637,
+      "tokens_seen": 2553348096
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.56989247311828e-05,
+      "loss": 1.222,
+      "theoretical_loss": 3.3913162234907572,
+      "tokens_seen": 2553479168
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.569090033702456e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.3913025131254977,
+      "tokens_seen": 2553610240
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.568287594286632e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.391288803660979,
+      "tokens_seen": 2553741312
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.567485154870808e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3912750950970962,
+      "tokens_seen": 2553872384
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.566682715454984e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3912613874337443,
+      "tokens_seen": 2554003456
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5658802760391597e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3912476806708174,
+      "tokens_seen": 2554134528
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.565077836623335e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.39123397480821,
+      "tokens_seen": 2554265600
+    },
+    {
+      "epoch": 0.55,
+      "objective/train/docs_used": 857322,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.310929536819458,
+      "objective/train/theoretical_loss": 3.3912202698458165,
+      "objective/train/tokens_used": 924855776,
+      "theoretical_loss": 3.3912202698458165,
+      "tokens_seen": 2554396672
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.564275397207511e-05,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3912202698458165,
+      "tokens_seen": 2554396672
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.563472957791687e-05,
+      "loss": 1.2913,
+      "theoretical_loss": 3.391206565783533,
+      "tokens_seen": 2554527744
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.562670518375863e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.391192862621253,
+      "tokens_seen": 2554658816
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.561868078960039e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.3911791603588712,
+      "tokens_seen": 2554789888
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.561065639544215e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3911654589962827,
+      "tokens_seen": 2554920960
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.560263200128391e-05,
+      "loss": 1.1823,
+      "theoretical_loss": 3.391151758533382,
+      "tokens_seen": 2555052032
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.559460760712567e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.3911380589700646,
+      "tokens_seen": 2555183104
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.558658321296742e-05,
+      "loss": 1.1902,
+      "theoretical_loss": 3.3911243603062244,
+      "tokens_seen": 2555314176
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.557855881880918e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.391110662541757,
+      "tokens_seen": 2555445248
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.557053442465094e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.391096965676556,
+      "tokens_seen": 2555576320
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.55625100304927e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.391083269710518,
+      "tokens_seen": 2555707392
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.555448563633446e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.391069574643536,
+      "tokens_seen": 2555838464
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.554646124217622e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3910558804755064,
+      "tokens_seen": 2555969536
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.553843684801798e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.3910421872063234,
+      "tokens_seen": 2556100608
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5530412453859735e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.391028494835882,
+      "tokens_seen": 2556231680
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5522388059701495e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.3910148033640772,
+      "tokens_seen": 2556362752
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5514363665543255e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.391001112790804,
+      "tokens_seen": 2556493824
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5506339271385014e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.390987423115957,
+      "tokens_seen": 2556624896
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5498314877226774e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3909737343394313,
+      "tokens_seen": 2556755968
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5490290483068534e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.3909600464611223,
+      "tokens_seen": 2556887040
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5482266088910294e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3909463594809246,
+      "tokens_seen": 2557018112
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5474241694752054e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3909326733987335,
+      "tokens_seen": 2557149184
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.546621730059381e-05,
+      "loss": 1.1567,
+      "theoretical_loss": 3.3909189882144437,
+      "tokens_seen": 2557280256
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.545819290643557e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.390905303927951,
+      "tokens_seen": 2557411328
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.545016851227733e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3908916205391497,
+      "tokens_seen": 2557542400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.544214411811909e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.3908779380479355,
+      "tokens_seen": 2557673472
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.543411972396085e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.3908642564542033,
+      "tokens_seen": 2557804544
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.542609532980261e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.390850575757848,
+      "tokens_seen": 2557935616
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.541807093564437e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.3908368959587647,
+      "tokens_seen": 2558066688
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.541004654148612e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.3908232170568495,
+      "tokens_seen": 2558197760
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.540202214732788e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.3908095390519963,
+      "tokens_seen": 2558328832
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.539399775316964e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.3907958619441017,
+      "tokens_seen": 2558459904
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.53859733590114e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.39078218573306,
+      "tokens_seen": 2558590976
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.537794896485316e-05,
+      "loss": 1.3115,
+      "theoretical_loss": 3.3907685104187664,
+      "tokens_seen": 2558722048
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.536992457069492e-05,
+      "loss": 1.1758,
+      "theoretical_loss": 3.390754836001116,
+      "tokens_seen": 2558853120
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.536190017653667e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.3907411624800052,
+      "tokens_seen": 2558984192
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.535387578237843e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.3907274898553283,
+      "tokens_seen": 2559115264
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.534585138822019e-05,
+      "loss": 1.3029,
+      "theoretical_loss": 3.390713818126981,
+      "tokens_seen": 2559246336
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.533782699406195e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.390700147294859,
+      "tokens_seen": 2559377408
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.532980259990371e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.3906864773588565,
+      "tokens_seen": 2559508480
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.532177820574547e-05,
+      "loss": 1.314,
+      "theoretical_loss": 3.39067280831887,
+      "tokens_seen": 2559639552
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5313753811587225e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.3906591401747943,
+      "tokens_seen": 2559770624
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5305729417428985e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.3906454729265247,
+      "tokens_seen": 2559901696
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5297705023270745e-05,
+      "loss": 1.1649,
+      "theoretical_loss": 3.390631806573958,
+      "tokens_seen": 2560032768
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5289680629112505e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.3906181411169873,
+      "tokens_seen": 2560163840
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5281656234954265e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.3906044765555103,
+      "tokens_seen": 2560294912
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5273631840796025e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.390590812889421,
+      "tokens_seen": 2560425984
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.526560744663778e-05,
+      "loss": 1.1914,
+      "theoretical_loss": 3.3905771501186157,
+      "tokens_seen": 2560557056
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.525758305247954e-05,
+      "loss": 1.3135,
+      "theoretical_loss": 3.3905634882429894,
+      "tokens_seen": 2560688128
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.52495586583213e-05,
+      "loss": 1.1403,
+      "theoretical_loss": 3.390549827262438,
+      "tokens_seen": 2560819200
+    },
+    {
+      "epoch": 0.55,
+      "objective/train/docs_used": 859666,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2541744709014893,
+      "objective/train/theoretical_loss": 3.390536167176857,
+      "objective/train/tokens_used": 931409376,
+      "theoretical_loss": 3.390536167176857,
+      "tokens_seen": 2560950272
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.524153426416306e-05,
+      "loss": 1.2816,
+      "theoretical_loss": 3.390536167176857,
+      "tokens_seen": 2560950272
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.523350987000482e-05,
+      "loss": 1.3103,
+      "theoretical_loss": 3.3905225079861414,
+      "tokens_seen": 2561081344
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.522548547584658e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.3905088496901876,
+      "tokens_seen": 2561212416
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.521746108168833e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.390495192288891,
+      "tokens_seen": 2561343488
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.520943668753009e-05,
+      "loss": 1.2768,
+      "theoretical_loss": 3.390481535782147,
+      "tokens_seen": 2561474560
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.520141229337185e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.390467880169851,
+      "tokens_seen": 2561605632
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.519338789921361e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.390454225451899,
+      "tokens_seen": 2561736704
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.518536350505537e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3904405716281873,
+      "tokens_seen": 2561867776
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.517733911089713e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.3904269186986102,
+      "tokens_seen": 2561998848
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.516931471673888e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.3904132666630646,
+      "tokens_seen": 2562129920
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.516129032258064e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3903996155214458,
+      "tokens_seen": 2562260992
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.51532659284224e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3903859652736488,
+      "tokens_seen": 2562392064
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.514524153426416e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.3903723159195707,
+      "tokens_seen": 2562523136
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.513721714010592e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.3903586674591066,
+      "tokens_seen": 2562654208
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.512919274594768e-05,
+      "loss": 1.2917,
+      "theoretical_loss": 3.390345019892152,
+      "tokens_seen": 2562785280
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.512116835178944e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3903313732186033,
+      "tokens_seen": 2562916352
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5113143957631196e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.3903177274383562,
+      "tokens_seen": 2563047424
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5105119563472956e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.390304082551306,
+      "tokens_seen": 2563178496
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5097095169314715e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.3902904385573494,
+      "tokens_seen": 2563309568
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5089070775156475e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.3902767954563817,
+      "tokens_seen": 2563440640
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5081046380998235e-05,
+      "loss": 1.2406,
+      "theoretical_loss": 3.390263153248299,
+      "tokens_seen": 2563571712
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5073021986839995e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.390249511932997,
+      "tokens_seen": 2563702784
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.5064997592681755e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.390235871510372,
+      "tokens_seen": 2563833856
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.505697319852351e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.39022223198032,
+      "tokens_seen": 2563964928
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.504894880436527e-05,
+      "loss": 1.1904,
+      "theoretical_loss": 3.3902085933427366,
+      "tokens_seen": 2564096000
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.504092441020703e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3901949555975177,
+      "tokens_seen": 2564227072
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.503290001604879e-05,
+      "loss": 1.3126,
+      "theoretical_loss": 3.3901813187445597,
+      "tokens_seen": 2564358144
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.502487562189055e-05,
+      "loss": 1.338,
+      "theoretical_loss": 3.3901676827837584,
+      "tokens_seen": 2564489216
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.501685122773231e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.39015404771501,
+      "tokens_seen": 2564620288
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.500882683357407e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3901404135382105,
+      "tokens_seen": 2564751360
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.500080243941583e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.3901267802532558,
+      "tokens_seen": 2564882432
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.499277804525758e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.390113147860042,
+      "tokens_seen": 2565013504
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.498475365109934e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.3900995163584655,
+      "tokens_seen": 2565144576
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.49767292569411e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.390085885748422,
+      "tokens_seen": 2565275648
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.496870486278286e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.390072256029808,
+      "tokens_seen": 2565406720
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.496068046862462e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.39005862720252,
+      "tokens_seen": 2565537792
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 4.495265607446638e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.390044999266453,
+      "tokens_seen": 2565668864
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.494463168030814e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3900313722215043,
+      "tokens_seen": 2565799936
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.493660728614989e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.39001774606757,
+      "tokens_seen": 2565931008
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.492858289199165e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3900041208045457,
+      "tokens_seen": 2566062080
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.492055849783341e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.389990496432328,
+      "tokens_seen": 2566193152
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.491253410367517e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3899768729508133,
+      "tokens_seen": 2566324224
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.490450970951693e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.389963250359898,
+      "tokens_seen": 2566455296
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.489648531535869e-05,
+      "loss": 1.1587,
+      "theoretical_loss": 3.389949628659478,
+      "tokens_seen": 2566586368
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.488846092120045e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3899360078494496,
+      "tokens_seen": 2566717440
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4880436527042206e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.389922387929709,
+      "tokens_seen": 2566848512
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4872412132883966e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.3899087689001535,
+      "tokens_seen": 2566979584
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4864387738725726e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.3898951507606787,
+      "tokens_seen": 2567110656
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4856363344567486e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.389881533511181,
+      "tokens_seen": 2567241728
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4848338950409245e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.389867917151557,
+      "tokens_seen": 2567372800
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 861842,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3470691442489624,
+      "objective/train/theoretical_loss": 3.3898543016817033,
+      "objective/train/tokens_used": 937962976,
+      "theoretical_loss": 3.3898543016817033,
+      "tokens_seen": 2567503872
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4840314556251005e-05,
+      "loss": 1.3039,
+      "theoretical_loss": 3.3898543016817033,
+      "tokens_seen": 2567503872
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4832290162092765e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.3898406871015156,
+      "tokens_seen": 2567634944
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4824265767934525e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.389827073410891,
+      "tokens_seen": 2567766016
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.481624137377628e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.389813460609726,
+      "tokens_seen": 2567897088
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.480821697961804e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.389799848697917,
+      "tokens_seen": 2568028160
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.48001925854598e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.3897862376753602,
+      "tokens_seen": 2568159232
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.479216819130156e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3897726275419524,
+      "tokens_seen": 2568290304
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.478414379714332e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.38975901829759,
+      "tokens_seen": 2568421376
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.477611940298508e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.3897454099421696,
+      "tokens_seen": 2568552448
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.476809500882684e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.389731802475588,
+      "tokens_seen": 2568683520
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.476007061466859e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3897181958977414,
+      "tokens_seen": 2568814592
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.475204622051035e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3897045902085265,
+      "tokens_seen": 2568945664
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.474402182635211e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.3896909854078405,
+      "tokens_seen": 2569076736
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.473599743219387e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.3896773814955794,
+      "tokens_seen": 2569207808
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.472797303803563e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.38966377847164,
+      "tokens_seen": 2569338880
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.471994864387739e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.3896501763359193,
+      "tokens_seen": 2569469952
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.471192424971915e-05,
+      "loss": 1.3118,
+      "theoretical_loss": 3.3896365750883137,
+      "tokens_seen": 2569601024
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.47038998555609e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.38962297472872,
+      "tokens_seen": 2569732096
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.469587546140266e-05,
+      "loss": 1.2727,
+      "theoretical_loss": 3.3896093752570344,
+      "tokens_seen": 2569863168
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.468785106724442e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.389595776673155,
+      "tokens_seen": 2569994240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.467982667308618e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.389582178976977,
+      "tokens_seen": 2570125312
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.467180227892794e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.389568582168398,
+      "tokens_seen": 2570256384
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.46637778847697e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.389554986247315,
+      "tokens_seen": 2570387456
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.465575349061146e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3895413912136245,
+      "tokens_seen": 2570518528
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.464772909645322e-05,
+      "loss": 1.3811,
+      "theoretical_loss": 3.389527797067223,
+      "tokens_seen": 2570649600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4639704702294976e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.389514203808008,
+      "tokens_seen": 2570780672
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4631680308136736e-05,
+      "loss": 1.2836,
+      "theoretical_loss": 3.389500611435876,
+      "tokens_seen": 2570911744
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4623655913978496e-05,
+      "loss": 1.1584,
+      "theoretical_loss": 3.389487019950724,
+      "tokens_seen": 2571042816
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4615631519820256e-05,
+      "loss": 1.283,
+      "theoretical_loss": 3.389473429352449,
+      "tokens_seen": 2571173888
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4607607125662015e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.3894598396409474,
+      "tokens_seen": 2571304960
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4599582731503775e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3894462508161167,
+      "tokens_seen": 2571436032
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4591558337345535e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.389432662877854,
+      "tokens_seen": 2571567104
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.458353394318729e-05,
+      "loss": 1.2998,
+      "theoretical_loss": 3.389419075826056,
+      "tokens_seen": 2571698176
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.457550954902905e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.389405489660619,
+      "tokens_seen": 2571829248
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.456748515487081e-05,
+      "loss": 1.2782,
+      "theoretical_loss": 3.3893919043814416,
+      "tokens_seen": 2571960320
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.455946076071257e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3893783199884195,
+      "tokens_seen": 2572091392
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.455143636655433e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.3893647364814496,
+      "tokens_seen": 2572222464
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.454341197239609e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.38935115386043,
+      "tokens_seen": 2572353536
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.453538757823785e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.3893375721252577,
+      "tokens_seen": 2572484608
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.452736318407961e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.3893239912758286,
+      "tokens_seen": 2572615680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.451933878992136e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.389310411312041,
+      "tokens_seen": 2572746752
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.451131439576312e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.3892968322337915,
+      "tokens_seen": 2572877824
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.450329000160488e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.3892832540409774,
+      "tokens_seen": 2573008896
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.449526560744664e-05,
+      "loss": 1.1237,
+      "theoretical_loss": 3.3892696767334956,
+      "tokens_seen": 2573139968
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.44872412132884e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.389256100311244,
+      "tokens_seen": 2573271040
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.447921681913016e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.3892425247741187,
+      "tokens_seen": 2573402112
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.447119242497192e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.389228950122018,
+      "tokens_seen": 2573533184
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4463168030813673e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.3892153763548385,
+      "tokens_seen": 2573664256
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.445514363665543e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.389201803472478,
+      "tokens_seen": 2573795328
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.444711924249719e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.389188231474833,
+      "tokens_seen": 2573926400
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 864102,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1688669919967651,
+      "objective/train/theoretical_loss": 3.389174660361801,
+      "objective/train/tokens_used": 944516576,
+      "theoretical_loss": 3.389174660361801,
+      "tokens_seen": 2574057472
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.443909484833895e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.389174660361801,
+      "tokens_seen": 2574057472
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.443107045418071e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.3891610901332796,
+      "tokens_seen": 2574188544
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.442304606002247e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.389147520789166,
+      "tokens_seen": 2574319616
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.441502166586423e-05,
+      "loss": 1.2968,
+      "theoretical_loss": 3.3891339523293573,
+      "tokens_seen": 2574450688
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4406997271705986e-05,
+      "loss": 1.2939,
+      "theoretical_loss": 3.389120384753751,
+      "tokens_seen": 2574581760
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4398972877547746e-05,
+      "loss": 1.2436,
+      "theoretical_loss": 3.389106818062245,
+      "tokens_seen": 2574712832
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4390948483389506e-05,
+      "loss": 1.1794,
+      "theoretical_loss": 3.389093252254736,
+      "tokens_seen": 2574843904
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4382924089231266e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.3890796873311215,
+      "tokens_seen": 2574974976
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4374899695073026e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.3890661232912995,
+      "tokens_seen": 2575106048
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4366875300914786e-05,
+      "loss": 1.2996,
+      "theoretical_loss": 3.3890525601351666,
+      "tokens_seen": 2575237120
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4358850906756545e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.3890389978626208,
+      "tokens_seen": 2575368192
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4350826512598305e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.389025436473559,
+      "tokens_seen": 2575499264
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.434280211844006e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3890118759678796,
+      "tokens_seen": 2575630336
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.433477772428182e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.388998316345479,
+      "tokens_seen": 2575761408
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.432675333012358e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.3889847576062557,
+      "tokens_seen": 2575892480
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.431872893596534e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.3889711997501073,
+      "tokens_seen": 2576023552
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.43107045418071e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.388957642776931,
+      "tokens_seen": 2576154624
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.430268014764886e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.388944086686623,
+      "tokens_seen": 2576285696
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.429465575349062e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.388930531479083,
+      "tokens_seen": 2576416768
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.428663135933237e-05,
+      "loss": 1.1599,
+      "theoretical_loss": 3.3889169771542083,
+      "tokens_seen": 2576547840
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.427860696517413e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.388903423711896,
+      "tokens_seen": 2576678912
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.427058257101589e-05,
+      "loss": 1.3422,
+      "theoretical_loss": 3.3888898711520428,
+      "tokens_seen": 2576809984
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.426255817685765e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.388876319474548,
+      "tokens_seen": 2576941056
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.425453378269941e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.3888627686793082,
+      "tokens_seen": 2577072128
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.424650938854117e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.388849218766222,
+      "tokens_seen": 2577203200
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.423848499438293e-05,
+      "loss": 1.3372,
+      "theoretical_loss": 3.3888356697351867,
+      "tokens_seen": 2577334272
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4230460600224684e-05,
+      "loss": 1.2136,
+      "theoretical_loss": 3.3888221215860996,
+      "tokens_seen": 2577465344
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4222436206066444e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3888085743188587,
+      "tokens_seen": 2577596416
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4214411811908203e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3887950279333623,
+      "tokens_seen": 2577727488
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.420638741774996e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.3887814824295073,
+      "tokens_seen": 2577858560
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.419836302359172e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.388767937807192,
+      "tokens_seen": 2577989632
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.419033862943348e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3887543940663143,
+      "tokens_seen": 2578120704
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.418231423527524e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.388740851206772,
+      "tokens_seen": 2578251776
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4174289841117e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.388727309228462,
+      "tokens_seen": 2578382848
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4166265446958756e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.388713768131284,
+      "tokens_seen": 2578513920
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4158241052800516e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.388700227915135,
+      "tokens_seen": 2578644992
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4150216658642276e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.388686688579912,
+      "tokens_seen": 2578776064
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4142192264484036e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3886731501255136,
+      "tokens_seen": 2578907136
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4134167870325796e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3886596125518382,
+      "tokens_seen": 2579038208
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4126143476167556e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3886460758587837,
+      "tokens_seen": 2579169280
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.4118119082009316e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.3886325400462476,
+      "tokens_seen": 2579300352
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.411009468785107e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.3886190051141276,
+      "tokens_seen": 2579431424
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.410207029369283e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.3886054710623226,
+      "tokens_seen": 2579562496
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.409404589953459e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.3885919378907294,
+      "tokens_seen": 2579693568
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.408602150537635e-05,
+      "loss": 1.2771,
+      "theoretical_loss": 3.3885784055992474,
+      "tokens_seen": 2579824640
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.407799711121811e-05,
+      "loss": 1.3179,
+      "theoretical_loss": 3.3885648741877743,
+      "tokens_seen": 2579955712
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.406997271705987e-05,
+      "loss": 1.2996,
+      "theoretical_loss": 3.388551343656207,
+      "tokens_seen": 2580086784
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.406194832290163e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3885378140044455,
+      "tokens_seen": 2580217856
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.405392392874339e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3885242852323865,
+      "tokens_seen": 2580348928
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.404589953458514e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.3885107573399282,
+      "tokens_seen": 2580480000
+    },
+    {
+      "epoch": 0.56,
+      "objective/train/docs_used": 866470,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.271082878112793,
+      "objective/train/theoretical_loss": 3.388497230326969,
+      "objective/train/tokens_used": 951070176,
+      "theoretical_loss": 3.388497230326969,
+      "tokens_seen": 2580611072
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.40378751404269e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.388497230326969,
+      "tokens_seen": 2580611072
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.402985074626866e-05,
+      "loss": 1.266,
+      "theoretical_loss": 3.3884837041934075,
+      "tokens_seen": 2580742144
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.402182635211042e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.3884701789391416,
+      "tokens_seen": 2580873216
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.401380195795218e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.388456654564069,
+      "tokens_seen": 2581004288
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.400577756379394e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3884431310680885,
+      "tokens_seen": 2581135360
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3997753169635694e-05,
+      "loss": 1.2819,
+      "theoretical_loss": 3.388429608451098,
+      "tokens_seen": 2581266432
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3989728775477454e-05,
+      "loss": 1.211,
+      "theoretical_loss": 3.3884160867129953,
+      "tokens_seen": 2581397504
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3981704381319214e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.38840256585368,
+      "tokens_seen": 2581528576
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3973679987160974e-05,
+      "loss": 1.3059,
+      "theoretical_loss": 3.3883890458730495,
+      "tokens_seen": 2581659648
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3965655593002733e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.388375526771002,
+      "tokens_seen": 2581790720
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.395763119884449e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.3883620085474355,
+      "tokens_seen": 2581921792
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3949606804686246e-05,
+      "loss": 1.1575,
+      "theoretical_loss": 3.3883484912022492,
+      "tokens_seen": 2582052864
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 4.3941582410528006e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.388334974735341,
+      "tokens_seen": 2582183936
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3933558016369766e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.388321459146609,
+      "tokens_seen": 2582315008
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3925533622211526e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.388307944435952,
+      "tokens_seen": 2582446080
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3917509228053286e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.3882944306032683,
+      "tokens_seen": 2582577152
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3909484833895046e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.3882809176484563,
+      "tokens_seen": 2582708224
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.39014604397368e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.388267405571414,
+      "tokens_seen": 2582839296
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.389343604557856e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3882538943720406,
+      "tokens_seen": 2582970368
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.388541165142032e-05,
+      "loss": 1.2188,
+      "theoretical_loss": 3.388240384050234,
+      "tokens_seen": 2583101440
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.387738725726208e-05,
+      "loss": 1.3271,
+      "theoretical_loss": 3.388226874605893,
+      "tokens_seen": 2583232512
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.386936286310384e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.388213366038916,
+      "tokens_seen": 2583363584
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.38613384689456e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.3881998583492012,
+      "tokens_seen": 2583494656
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.385331407478735e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3881863515366475,
+      "tokens_seen": 2583625728
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.384528968062911e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.3881728456011535,
+      "tokens_seen": 2583756800
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.383726528647087e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.388159340542617,
+      "tokens_seen": 2583887872
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.382924089231263e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.3881458363609376,
+      "tokens_seen": 2584018944
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.382121649815439e-05,
+      "loss": 1.1682,
+      "theoretical_loss": 3.3881323330560136,
+      "tokens_seen": 2584150016
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.381319210399615e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.3881188306277434,
+      "tokens_seen": 2584281088
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3805167709837904e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3881053290760255,
+      "tokens_seen": 2584412160
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3797143315679664e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.388091828400759,
+      "tokens_seen": 2584543232
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3789118921521424e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.388078328601842,
+      "tokens_seen": 2584674304
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3781094527363184e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3880648296791733,
+      "tokens_seen": 2584805376
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3773070133204944e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.388051331632652,
+      "tokens_seen": 2584936448
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3765045739046704e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.388037834462177,
+      "tokens_seen": 2585067520
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.375702134488846e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.388024338167646,
+      "tokens_seen": 2585198592
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.374899695073022e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.3880108427489586,
+      "tokens_seen": 2585329664
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.374097255657198e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.3879973482060137,
+      "tokens_seen": 2585460736
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.373294816241374e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.387983854538709,
+      "tokens_seen": 2585591808
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.37249237682555e-05,
+      "loss": 1.2632,
+      "theoretical_loss": 3.3879703617469445,
+      "tokens_seen": 2585722880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.371689937409726e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.387956869830618,
+      "tokens_seen": 2585853952
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3708874979939017e-05,
+      "loss": 1.2106,
+      "theoretical_loss": 3.3879433787896294,
+      "tokens_seen": 2585985024
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3700850585780776e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.387929888623877,
+      "tokens_seen": 2586116096
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.369282619162253e-05,
+      "loss": 1.283,
+      "theoretical_loss": 3.387916399333259,
+      "tokens_seen": 2586247168
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.368480179746429e-05,
+      "loss": 1.2651,
+      "theoretical_loss": 3.3879029109176755,
+      "tokens_seen": 2586378240
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.367677740330605e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3878894233770245,
+      "tokens_seen": 2586509312
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.366875300914781e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3878759367112057,
+      "tokens_seen": 2586640384
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.366072861498957e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.387862450920117,
+      "tokens_seen": 2586771456
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.365270422083133e-05,
+      "loss": 1.2869,
+      "theoretical_loss": 3.3878489660036584,
+      "tokens_seen": 2586902528
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.364467982667309e-05,
+      "loss": 1.32,
+      "theoretical_loss": 3.3878354819617282,
+      "tokens_seen": 2587033600
+    },
+    {
+      "epoch": 0.57,
+      "objective/train/docs_used": 868686,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2676774263381958,
+      "objective/train/theoretical_loss": 3.3878219987942253,
+      "objective/train/tokens_used": 957623776,
+      "theoretical_loss": 3.3878219987942253,
+      "tokens_seen": 2587164672
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.363665543251484e-05,
+      "loss": 1.2942,
+      "theoretical_loss": 3.3878219987942253,
+      "tokens_seen": 2587164672
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.36286310383566e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.387808516501049,
+      "tokens_seen": 2587295744
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.362060664419836e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.3877950350820987,
+      "tokens_seen": 2587426816
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.361258225004012e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.387781554537273,
+      "tokens_seen": 2587557888
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.360455785588188e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3877680748664707,
+      "tokens_seen": 2587688960
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.359653346172364e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.387754596069591,
+      "tokens_seen": 2587820032
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.35885090675654e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3877411181465336,
+      "tokens_seen": 2587951104
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.358048467340716e-05,
+      "loss": 1.1686,
+      "theoretical_loss": 3.3877276410971966,
+      "tokens_seen": 2588082176
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3572460279248915e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3877141649214795,
+      "tokens_seen": 2588213248
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3564435885090675e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.387700689619282,
+      "tokens_seen": 2588344320
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3556411490932434e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3876872151905024,
+      "tokens_seen": 2588475392
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3548387096774194e-05,
+      "loss": 1.3014,
+      "theoretical_loss": 3.387673741635041,
+      "tokens_seen": 2588606464
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3540362702615954e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.3876602689527955,
+      "tokens_seen": 2588737536
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3532338308457714e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.387646797143666,
+      "tokens_seen": 2588868608
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3524313914299474e-05,
+      "loss": 1.1653,
+      "theoretical_loss": 3.3876333262075518,
+      "tokens_seen": 2588999680
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.351628952014123e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.387619856144352,
+      "tokens_seen": 2589130752
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.350826512598299e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3876063869539648,
+      "tokens_seen": 2589261824
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.350024073182475e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.387592918636291,
+      "tokens_seen": 2589392896
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.349221633766651e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.38757945119123,
+      "tokens_seen": 2589523968
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.348419194350827e-05,
+      "loss": 1.1906,
+      "theoretical_loss": 3.3875659846186794,
+      "tokens_seen": 2589655040
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.347616754935003e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.38755251891854,
+      "tokens_seen": 2589786112
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3468143155191787e-05,
+      "loss": 1.1859,
+      "theoretical_loss": 3.3875390540907104,
+      "tokens_seen": 2589917184
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.346011876103354e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.3875255901350902,
+      "tokens_seen": 2590048256
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.34520943668753e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.387512127051579,
+      "tokens_seen": 2590179328
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.344406997271706e-05,
+      "loss": 1.2994,
+      "theoretical_loss": 3.387498664840076,
+      "tokens_seen": 2590310400
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.343604557855882e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.3874852035004803,
+      "tokens_seen": 2590441472
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.342802118440058e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.387471743032691,
+      "tokens_seen": 2590572544
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.341999679024234e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.387458283436609,
+      "tokens_seen": 2590703616
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.34119723960841e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.3874448247121327,
+      "tokens_seen": 2590834688
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.340394800192586e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.387431366859161,
+      "tokens_seen": 2590965760
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.339592360776761e-05,
+      "loss": 1.304,
+      "theoretical_loss": 3.3874179098775947,
+      "tokens_seen": 2591096832
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.338789921360937e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.3874044537673327,
+      "tokens_seen": 2591227904
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.337987481945113e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.387390998528274,
+      "tokens_seen": 2591358976
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.337185042529289e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3873775441603193,
+      "tokens_seen": 2591490048
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.336382603113465e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3873640906633673,
+      "tokens_seen": 2591621120
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.335580163697641e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.3873506380373177,
+      "tokens_seen": 2591752192
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.334777724281817e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.3873371862820703,
+      "tokens_seen": 2591883264
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3339752848659925e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.387323735397524,
+      "tokens_seen": 2592014336
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3331728454501685e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.3873102853835793,
+      "tokens_seen": 2592145408
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3323704060343445e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.3872968362401354,
+      "tokens_seen": 2592276480
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3315679666185204e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.3872833879670923,
+      "tokens_seen": 2592407552
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3307655272026964e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.387269940564349,
+      "tokens_seen": 2592538624
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3299630877868724e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.387256494031806,
+      "tokens_seen": 2592669696
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3291606483710484e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.387243048369362,
+      "tokens_seen": 2592800768
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.328358208955224e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.387229603576918,
+      "tokens_seen": 2592931840
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3275557695394e-05,
+      "loss": 1.2817,
+      "theoretical_loss": 3.3872161596543724,
+      "tokens_seen": 2593062912
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.326753330123576e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.3872027166016263,
+      "tokens_seen": 2593193984
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.325950890707752e-05,
+      "loss": 1.3048,
+      "theoretical_loss": 3.387189274418578,
+      "tokens_seen": 2593325056
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.325148451291928e-05,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3871758331051285,
+      "tokens_seen": 2593456128
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.324346011876104e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.387162392661177,
+      "tokens_seen": 2593587200
+    },
+    {
+      "epoch": 0.57,
+      "objective/train/docs_used": 870893,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2980492115020752,
+      "objective/train/theoretical_loss": 3.3871489530866232,
+      "objective/train/tokens_used": 964177376,
+      "theoretical_loss": 3.3871489530866232,
+      "tokens_seen": 2593718272
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.32354357246028e-05,
+      "loss": 1.2702,
+      "theoretical_loss": 3.3871489530866232,
+      "tokens_seen": 2593718272
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.322741133044456e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3871355143813675,
+      "tokens_seen": 2593849344
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.321938693628631e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.3871220765453094,
+      "tokens_seen": 2593980416
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.321136254212807e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3871086395783485,
+      "tokens_seen": 2594111488
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.320333814796983e-05,
+      "loss": 1.279,
+      "theoretical_loss": 3.3870952034803854,
+      "tokens_seen": 2594242560
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.319531375381159e-05,
+      "loss": 1.1722,
+      "theoretical_loss": 3.3870817682513192,
+      "tokens_seen": 2594373632
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.318728935965335e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3870683338910506,
+      "tokens_seen": 2594504704
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.317926496549511e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3870549003994785,
+      "tokens_seen": 2594635776
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.317124057133687e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.387041467776504,
+      "tokens_seen": 2594766848
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.316321617717862e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3870280360220266,
+      "tokens_seen": 2594897920
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.315519178302038e-05,
+      "loss": 1.2795,
+      "theoretical_loss": 3.387014605135946,
+      "tokens_seen": 2595028992
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.314716738886214e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.3870011751181623,
+      "tokens_seen": 2595160064
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.31391429947039e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3869877459685758,
+      "tokens_seen": 2595291136
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.313111860054566e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3869743176870863,
+      "tokens_seen": 2595422208
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.312309420638742e-05,
+      "loss": 1.298,
+      "theoretical_loss": 3.386960890273594,
+      "tokens_seen": 2595553280
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.311506981222918e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.386947463727999,
+      "tokens_seen": 2595684352
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.310704541807094e-05,
+      "loss": 1.2915,
+      "theoretical_loss": 3.386934038050201,
+      "tokens_seen": 2595815424
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3099021023912695e-05,
+      "loss": 1.2106,
+      "theoretical_loss": 3.3869206132401004,
+      "tokens_seen": 2595946496
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3090996629754455e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.386907189297597,
+      "tokens_seen": 2596077568
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3082972235596215e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.3868937662225918,
+      "tokens_seen": 2596208640
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3074947841437975e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.3868803440149837,
+      "tokens_seen": 2596339712
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3066923447279734e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.386866922674674,
+      "tokens_seen": 2596470784
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3058899053121494e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.3868535022015624,
+      "tokens_seen": 2596601856
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.3050874658963254e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.386840082595549,
+      "tokens_seen": 2596732928
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.304285026480501e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.386826663856534,
+      "tokens_seen": 2596864000
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.303482587064677e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.386813245984418,
+      "tokens_seen": 2596995072
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.302680147648853e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3867998289791004,
+      "tokens_seen": 2597126144
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.301877708233029e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3867864128404825,
+      "tokens_seen": 2597257216
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.301075268817205e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.386772997568464,
+      "tokens_seen": 2597388288
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.300272829401381e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.386759583162945,
+      "tokens_seen": 2597519360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.299470389985557e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.3867461696238266,
+      "tokens_seen": 2597650432
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.298667950569732e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.3867327569510084,
+      "tokens_seen": 2597781504
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.297865511153908e-05,
+      "loss": 1.3284,
+      "theoretical_loss": 3.386719345144391,
+      "tokens_seen": 2597912576
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.297063071738084e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.3867059342038752,
+      "tokens_seen": 2598043648
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.29626063232226e-05,
+      "loss": 1.2961,
+      "theoretical_loss": 3.3866925241293604,
+      "tokens_seen": 2598174720
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.295458192906436e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.3866791149207476,
+      "tokens_seen": 2598305792
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.294655753490612e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.3866657065779373,
+      "tokens_seen": 2598436864
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.293853314074788e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.38665229910083,
+      "tokens_seen": 2598567936
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.293050874658964e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3866388924893256,
+      "tokens_seen": 2598699008
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.292248435243139e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.386625486743325,
+      "tokens_seen": 2598830080
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.291445995827315e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.3866120818627286,
+      "tokens_seen": 2598961152
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.290643556411491e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.386598677847437,
+      "tokens_seen": 2599092224
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.289841116995667e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.3865852746973504,
+      "tokens_seen": 2599223296
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.289038677579843e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.3865718724123695,
+      "tokens_seen": 2599354368
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.288236238164019e-05,
+      "loss": 1.295,
+      "theoretical_loss": 3.386558470992395,
+      "tokens_seen": 2599485440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.287433798748195e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.386545070437327,
+      "tokens_seen": 2599616512
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2866313593323705e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.3865316707470665,
+      "tokens_seen": 2599747584
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2858289199165465e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.386518271921514,
+      "tokens_seen": 2599878656
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2850264805007225e-05,
+      "loss": 1.3145,
+      "theoretical_loss": 3.38650487396057,
+      "tokens_seen": 2600009728
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2842240410848985e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.386491476864135,
+      "tokens_seen": 2600140800
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 872942,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.205723524093628,
+      "objective/train/theoretical_loss": 3.38647808063211,
+      "objective/train/tokens_used": 970730976,
+      "theoretical_loss": 3.38647808063211,
+      "tokens_seen": 2600271872
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2834216016690745e-05,
+      "loss": 1.2688,
+      "theoretical_loss": 3.38647808063211,
+      "tokens_seen": 2600271872
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2826191622532505e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.3864646852643956,
+      "tokens_seen": 2600402944
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2818167228374264e-05,
+      "loss": 1.3019,
+      "theoretical_loss": 3.3864512907608924,
+      "tokens_seen": 2600534016
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.281014283421602e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.386437897121501,
+      "tokens_seen": 2600665088
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.280211844005778e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.386424504346122,
+      "tokens_seen": 2600796160
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.279409404589954e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.3864111124346565,
+      "tokens_seen": 2600927232
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.27860696517413e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.3863977213870053,
+      "tokens_seen": 2601058304
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.277804525758306e-05,
+      "loss": 1.3064,
+      "theoretical_loss": 3.3863843312030686,
+      "tokens_seen": 2601189376
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.277002086342482e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3863709418827472,
+      "tokens_seen": 2601320448
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.276199646926658e-05,
+      "loss": 1.232,
+      "theoretical_loss": 3.3863575534259427,
+      "tokens_seen": 2601451520
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.275397207510834e-05,
+      "loss": 1.1938,
+      "theoretical_loss": 3.386344165832555,
+      "tokens_seen": 2601582592
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.274594768095009e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3863307791024853,
+      "tokens_seen": 2601713664
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.273792328679185e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3863173932356343,
+      "tokens_seen": 2601844736
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.272989889263361e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.386304008231903,
+      "tokens_seen": 2601975808
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.272187449847537e-05,
+      "loss": 1.1808,
+      "theoretical_loss": 3.3862906240911927,
+      "tokens_seen": 2602106880
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.271385010431713e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.3862772408134036,
+      "tokens_seen": 2602237952
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.270582571015889e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.386263858398437,
+      "tokens_seen": 2602369024
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.269780131600065e-05,
+      "loss": 1.2349,
+      "theoretical_loss": 3.386250476846193,
+      "tokens_seen": 2602500096
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.26897769218424e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.3862370961565738,
+      "tokens_seen": 2602631168
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.268175252768416e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.38622371632948,
+      "tokens_seen": 2602762240
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.267372813352592e-05,
+      "loss": 1.2757,
+      "theoretical_loss": 3.386210337364812,
+      "tokens_seen": 2602893312
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.266570373936768e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.3861969592624708,
+      "tokens_seen": 2603024384
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.265767934520944e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.3861835820223583,
+      "tokens_seen": 2603155456
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.26496549510512e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.3861702056443743,
+      "tokens_seen": 2603286528
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2641630556892955e-05,
+      "loss": 1.2101,
+      "theoretical_loss": 3.386156830128421,
+      "tokens_seen": 2603417600
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2633606162734715e-05,
+      "loss": 1.3147,
+      "theoretical_loss": 3.3861434554743983,
+      "tokens_seen": 2603548672
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2625581768576475e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.386130081682208,
+      "tokens_seen": 2603679744
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2617557374418235e-05,
+      "loss": 1.1873,
+      "theoretical_loss": 3.386116708751752,
+      "tokens_seen": 2603810816
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2609532980259995e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.3861033366829294,
+      "tokens_seen": 2603941888
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2601508586101755e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3860899654756427,
+      "tokens_seen": 2604072960
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.259348419194351e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.386076595129793,
+      "tokens_seen": 2604204032
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.258545979778527e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.386063225645281,
+      "tokens_seen": 2604335104
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.257743540362703e-05,
+      "loss": 1.3096,
+      "theoretical_loss": 3.386049857022008,
+      "tokens_seen": 2604466176
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.256941100946879e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.386036489259875,
+      "tokens_seen": 2604597248
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.256138661531055e-05,
+      "loss": 1.3613,
+      "theoretical_loss": 3.3860231223587838,
+      "tokens_seen": 2604728320
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.255336222115231e-05,
+      "loss": 1.2984,
+      "theoretical_loss": 3.386009756318635,
+      "tokens_seen": 2604859392
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.254533782699406e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3859963911393303,
+      "tokens_seen": 2604990464
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.253731343283582e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.3859830268207705,
+      "tokens_seen": 2605121536
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.252928903867758e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.3859696633628573,
+      "tokens_seen": 2605252608
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.252126464451934e-05,
+      "loss": 1.3033,
+      "theoretical_loss": 3.3859563007654914,
+      "tokens_seen": 2605383680
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.25132402503611e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3859429390285745,
+      "tokens_seen": 2605514752
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.250521585620286e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.385929578152008,
+      "tokens_seen": 2605645824
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.249719146204461e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.385916218135693,
+      "tokens_seen": 2605776896
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.248916706788637e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.385902858979531,
+      "tokens_seen": 2605907968
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.248114267372813e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.3858895006834233,
+      "tokens_seen": 2606039040
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.247311827956989e-05,
+      "loss": 1.3231,
+      "theoretical_loss": 3.385876143247271,
+      "tokens_seen": 2606170112
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.246509388541165e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.3858627866709763,
+      "tokens_seen": 2606301184
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.245706949125341e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.3858494309544396,
+      "tokens_seen": 2606432256
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2449045097095166e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.385836076097563,
+      "tokens_seen": 2606563328
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2441020702936926e-05,
+      "loss": 1.3223,
+      "theoretical_loss": 3.3858227221002477,
+      "tokens_seen": 2606694400
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 875060,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.30265474319458,
+      "objective/train/theoretical_loss": 3.3858093689623954,
+      "objective/train/tokens_used": 977284576,
+      "theoretical_loss": 3.3858093689623954,
+      "tokens_seen": 2606825472
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2432996308778686e-05,
+      "loss": 1.2996,
+      "theoretical_loss": 3.3858093689623954,
+      "tokens_seen": 2606825472
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2424971914620446e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.385796016683907,
+      "tokens_seen": 2606956544
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2416947520462206e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.385782665264685,
+      "tokens_seen": 2607087616
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2408923126303965e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3857693147046297,
+      "tokens_seen": 2607218688
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2400898732145725e-05,
+      "loss": 1.2818,
+      "theoretical_loss": 3.3857559650036433,
+      "tokens_seen": 2607349760
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.239287433798748e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.385742616161627,
+      "tokens_seen": 2607480832
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.238484994382924e-05,
+      "loss": 1.3662,
+      "theoretical_loss": 3.3857292681784834,
+      "tokens_seen": 2607611904
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2376825549671e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.385715921054113,
+      "tokens_seen": 2607742976
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.236880115551276e-05,
+      "loss": 1.3389,
+      "theoretical_loss": 3.385702574788417,
+      "tokens_seen": 2607874048
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.236077676135452e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.385689229381298,
+      "tokens_seen": 2608005120
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.235275236719628e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.385675884832658,
+      "tokens_seen": 2608136192
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.234472797303804e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.385662541142397,
+      "tokens_seen": 2608267264
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.233670357887979e-05,
+      "loss": 1.3356,
+      "theoretical_loss": 3.3856491983104178,
+      "tokens_seen": 2608398336
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.232867918472155e-05,
+      "loss": 1.2778,
+      "theoretical_loss": 3.385635856336622,
+      "tokens_seen": 2608529408
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.232065479056331e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.385622515220911,
+      "tokens_seen": 2608660480
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.231263039640507e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3856091749631867,
+      "tokens_seen": 2608791552
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.230460600224683e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.385595835563351,
+      "tokens_seen": 2608922624
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.229658160808859e-05,
+      "loss": 1.2836,
+      "theoretical_loss": 3.385582497021305,
+      "tokens_seen": 2609053696
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.228855721393035e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.385569159336951,
+      "tokens_seen": 2609184768
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.228053281977211e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3855558225101907,
+      "tokens_seen": 2609315840
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2272508425613864e-05,
+      "loss": 1.3068,
+      "theoretical_loss": 3.3855424865409254,
+      "tokens_seen": 2609446912
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2264484031455623e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3855291514290577,
+      "tokens_seen": 2609577984
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.225645963729738e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.385515817174489,
+      "tokens_seen": 2609709056
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.224843524313914e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.385502483777121,
+      "tokens_seen": 2609840128
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.22404108489809e-05,
+      "loss": 1.2957,
+      "theoretical_loss": 3.3854891512368557,
+      "tokens_seen": 2609971200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.223238645482266e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.385475819553595,
+      "tokens_seen": 2610102272
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.222436206066442e-05,
+      "loss": 1.2822,
+      "theoretical_loss": 3.3854624887272404,
+      "tokens_seen": 2610233344
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2216337666506176e-05,
+      "loss": 1.126,
+      "theoretical_loss": 3.3854491587576945,
+      "tokens_seen": 2610364416
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2208313272347936e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.385435829644859,
+      "tokens_seen": 2610495488
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2200288878189696e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.385422501388635,
+      "tokens_seen": 2610626560
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2192264484031456e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3854091739889256,
+      "tokens_seen": 2610757632
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2184240089873216e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.3853958474456323,
+      "tokens_seen": 2610888704
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2176215695714976e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3853825217586566,
+      "tokens_seen": 2611019776
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2168191301556736e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.3853691969279014,
+      "tokens_seen": 2611150848
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2160166907398495e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.3853558729532685,
+      "tokens_seen": 2611281920
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.215214251324025e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.385342549834659,
+      "tokens_seen": 2611412992
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.214411811908201e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.385329227571976,
+      "tokens_seen": 2611544064
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.213609372492377e-05,
+      "loss": 1.3004,
+      "theoretical_loss": 3.385315906165121,
+      "tokens_seen": 2611675136
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.212806933076553e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.385302585613996,
+      "tokens_seen": 2611806208
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.212004493660729e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3852892659185043,
+      "tokens_seen": 2611937280
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.211202054244905e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.3852759470785463,
+      "tokens_seen": 2612068352
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.210399614829081e-05,
+      "loss": 1.3084,
+      "theoretical_loss": 3.385262629094025,
+      "tokens_seen": 2612199424
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.209597175413256e-05,
+      "loss": 1.3577,
+      "theoretical_loss": 3.3852493119648424,
+      "tokens_seen": 2612330496
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.208794735997432e-05,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3852359956909006,
+      "tokens_seen": 2612461568
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.207992296581608e-05,
+      "loss": 1.2856,
+      "theoretical_loss": 3.385222680272102,
+      "tokens_seen": 2612592640
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.207189857165784e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.3852093657083486,
+      "tokens_seen": 2612723712
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.20638741774996e-05,
+      "loss": 1.2226,
+      "theoretical_loss": 3.3851960519995425,
+      "tokens_seen": 2612854784
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.205584978334136e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.385182739145586,
+      "tokens_seen": 2612985856
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.204782538918312e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3851694271463817,
+      "tokens_seen": 2613116928
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2039800995024874e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.385156116001831,
+      "tokens_seen": 2613248000
+    },
+    {
+      "epoch": 0.58,
+      "objective/train/docs_used": 877266,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2504823207855225,
+      "objective/train/theoretical_loss": 3.385142805711837,
+      "objective/train/tokens_used": 983838176,
+      "theoretical_loss": 3.385142805711837,
+      "tokens_seen": 2613379072
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2031776600866634e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.385142805711837,
+      "tokens_seen": 2613379072
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2023752206708394e-05,
+      "loss": 1.4098,
+      "theoretical_loss": 3.385129496276302,
+      "tokens_seen": 2613510144
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.2015727812550153e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3851161876951275,
+      "tokens_seen": 2613641216
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.200770341839191e-05,
+      "loss": 1.2825,
+      "theoretical_loss": 3.3851028799682163,
+      "tokens_seen": 2613772288
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.199967902423367e-05,
+      "loss": 1.1733,
+      "theoretical_loss": 3.385089573095471,
+      "tokens_seen": 2613903360
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.199165463007543e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.3850762670767933,
+      "tokens_seen": 2614034432
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.198363023591719e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.3850629619120864,
+      "tokens_seen": 2614165504
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1975605841758946e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.385049657601252,
+      "tokens_seen": 2614296576
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1967581447600706e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3850363541441926,
+      "tokens_seen": 2614427648
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1959557053442466e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.385023051540811,
+      "tokens_seen": 2614558720
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1951532659284226e-05,
+      "loss": 1.2611,
+      "theoretical_loss": 3.385009749791009,
+      "tokens_seen": 2614689792
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1943508265125986e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.3849964488946895,
+      "tokens_seen": 2614820864
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1935483870967746e-05,
+      "loss": 1.2846,
+      "theoretical_loss": 3.384983148851755,
+      "tokens_seen": 2614951936
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 4.1927459476809506e-05,
+      "loss": 1.3607,
+      "theoretical_loss": 3.3849698496621077,
+      "tokens_seen": 2615083008
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.191943508265126e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.3849565513256503,
+      "tokens_seen": 2615214080
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.191141068849302e-05,
+      "loss": 1.3292,
+      "theoretical_loss": 3.3849432538422852,
+      "tokens_seen": 2615345152
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.190338629433478e-05,
+      "loss": 1.3141,
+      "theoretical_loss": 3.3849299572119147,
+      "tokens_seen": 2615476224
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.189536190017654e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.384916661434442,
+      "tokens_seen": 2615607296
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.18873375060183e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.384903366509769,
+      "tokens_seen": 2615738368
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.187931311186006e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.384890072437799,
+      "tokens_seen": 2615869440
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.187128871770182e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.3848767792184336,
+      "tokens_seen": 2616000512
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.186326432354357e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.384863486851576,
+      "tokens_seen": 2616131584
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.185523992938533e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.384850195337129,
+      "tokens_seen": 2616262656
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.184721553522709e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.3848369046749944,
+      "tokens_seen": 2616393728
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.183919114106885e-05,
+      "loss": 1.28,
+      "theoretical_loss": 3.3848236148650757,
+      "tokens_seen": 2616524800
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.183116674691061e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.3848103259072753,
+      "tokens_seen": 2616655872
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.182314235275237e-05,
+      "loss": 1.288,
+      "theoretical_loss": 3.384797037801496,
+      "tokens_seen": 2616786944
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.181511795859413e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.38478375054764,
+      "tokens_seen": 2616918016
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.180709356443589e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.384770464145611,
+      "tokens_seen": 2617049088
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1799069170277644e-05,
+      "loss": 1.3076,
+      "theoretical_loss": 3.3847571785953106,
+      "tokens_seen": 2617180160
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1791044776119404e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.3847438938966423,
+      "tokens_seen": 2617311232
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1783020381961164e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.384730610049509,
+      "tokens_seen": 2617442304
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1774995987802923e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.3847173270538127,
+      "tokens_seen": 2617573376
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1766971593644683e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.3847040449094563,
+      "tokens_seen": 2617704448
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.175894719948644e-05,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3846907636163435,
+      "tokens_seen": 2617835520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.17509228053282e-05,
+      "loss": 1.3796,
+      "theoretical_loss": 3.384677483174376,
+      "tokens_seen": 2617966592
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1742898411169956e-05,
+      "loss": 1.303,
+      "theoretical_loss": 3.3846642035834575,
+      "tokens_seen": 2618097664
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1734874017011716e-05,
+      "loss": 1.3022,
+      "theoretical_loss": 3.3846509248434904,
+      "tokens_seen": 2618228736
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1726849622853476e-05,
+      "loss": 1.2702,
+      "theoretical_loss": 3.384637646954378,
+      "tokens_seen": 2618359808
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1718825228695236e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.384624369916023,
+      "tokens_seen": 2618490880
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1710800834536996e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.384611093728328,
+      "tokens_seen": 2618621952
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1702776440378756e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.384597818391196,
+      "tokens_seen": 2618753024
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1694752046220516e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.38458454390453,
+      "tokens_seen": 2618884096
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1686727652062276e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.3845712702682333,
+      "tokens_seen": 2619015168
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.167870325790403e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.384557997482209,
+      "tokens_seen": 2619146240
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.167067886374579e-05,
+      "loss": 1.297,
+      "theoretical_loss": 3.384544725546359,
+      "tokens_seen": 2619277312
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.166265446958755e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.3845314544605873,
+      "tokens_seen": 2619408384
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.165463007542931e-05,
+      "loss": 1.2692,
+      "theoretical_loss": 3.3845181842247967,
+      "tokens_seen": 2619539456
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.164660568127107e-05,
+      "loss": 1.2987,
+      "theoretical_loss": 3.38450491483889,
+      "tokens_seen": 2619670528
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.163858128711283e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3844916463027706,
+      "tokens_seen": 2619801600
+    },
+    {
+      "epoch": 0.59,
+      "objective/train/docs_used": 879580,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1872670650482178,
+      "objective/train/theoretical_loss": 3.3844783786163415,
+      "objective/train/tokens_used": 990391776,
+      "theoretical_loss": 3.3844783786163415,
+      "tokens_seen": 2619932672
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.163055689295459e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.3844783786163415,
+      "tokens_seen": 2619932672
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.162253249879634e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3844651117795053,
+      "tokens_seen": 2620063744
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.16145081046381e-05,
+      "loss": 1.3415,
+      "theoretical_loss": 3.3844518457921655,
+      "tokens_seen": 2620194816
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.160648371047986e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3844385806542254,
+      "tokens_seen": 2620325888
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.159845931632162e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3844253163655877,
+      "tokens_seen": 2620456960
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.159043492216338e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.384412052926156,
+      "tokens_seen": 2620588032
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.158241052800514e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.3843987903358332,
+      "tokens_seen": 2620719104
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.15743861338469e-05,
+      "loss": 1.3456,
+      "theoretical_loss": 3.3843855285945224,
+      "tokens_seen": 2620850176
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1566361739688654e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.3843722677021266,
+      "tokens_seen": 2620981248
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1558337345530414e-05,
+      "loss": 1.3381,
+      "theoretical_loss": 3.3843590076585492,
+      "tokens_seen": 2621112320
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1550312951372174e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.3843457484636943,
+      "tokens_seen": 2621243392
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1542288557213934e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3843324901174636,
+      "tokens_seen": 2621374464
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1534264163055694e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.3843192326197618,
+      "tokens_seen": 2621505536
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1526239768897453e-05,
+      "loss": 1.2981,
+      "theoretical_loss": 3.384305975970491,
+      "tokens_seen": 2621636608
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.151821537473921e-05,
+      "loss": 1.3132,
+      "theoretical_loss": 3.384292720169555,
+      "tokens_seen": 2621767680
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.151019098058097e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.3842794652168573,
+      "tokens_seen": 2621898752
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1502166586422726e-05,
+      "loss": 1.3326,
+      "theoretical_loss": 3.3842662111123007,
+      "tokens_seen": 2622029824
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1494142192264486e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.3842529578557894,
+      "tokens_seen": 2622160896
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1486117798106246e-05,
+      "loss": 1.3435,
+      "theoretical_loss": 3.3842397054472255,
+      "tokens_seen": 2622291968
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1478093403948006e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.3842264538865137,
+      "tokens_seen": 2622423040
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1470069009789766e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.384213203173556,
+      "tokens_seen": 2622554112
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1462044615631526e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3841999533082574,
+      "tokens_seen": 2622685184
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1454020221473286e-05,
+      "loss": 1.175,
+      "theoretical_loss": 3.38418670429052,
+      "tokens_seen": 2622816256
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.144599582731504e-05,
+      "loss": 1.2962,
+      "theoretical_loss": 3.384173456120248,
+      "tokens_seen": 2622947328
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.14379714331568e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.384160208797344,
+      "tokens_seen": 2623078400
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.142994703899856e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3841469623217124,
+      "tokens_seen": 2623209472
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.142192264484032e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.384133716693256,
+      "tokens_seen": 2623340544
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.141389825068208e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3841204719118787,
+      "tokens_seen": 2623471616
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.140587385652384e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.384107227977484,
+      "tokens_seen": 2623602688
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.13978494623656e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.3840939848899754,
+      "tokens_seen": 2623733760
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.138982506820735e-05,
+      "loss": 1.3031,
+      "theoretical_loss": 3.3840807426492563,
+      "tokens_seen": 2623864832
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.138180067404911e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.38406750125523,
+      "tokens_seen": 2623995904
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.137377627989087e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.384054260707801,
+      "tokens_seen": 2624126976
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.136575188573263e-05,
+      "loss": 1.3563,
+      "theoretical_loss": 3.384041021006872,
+      "tokens_seen": 2624258048
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.135772749157439e-05,
+      "loss": 1.3663,
+      "theoretical_loss": 3.384027782152347,
+      "tokens_seen": 2624389120
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.134970309741615e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.3840145441441294,
+      "tokens_seen": 2624520192
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.134167870325791e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.384001306982123,
+      "tokens_seen": 2624651264
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.133365430909967e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.383988070666231,
+      "tokens_seen": 2624782336
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1325629914941424e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.3839748351963577,
+      "tokens_seen": 2624913408
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1317605520783184e-05,
+      "loss": 1.2645,
+      "theoretical_loss": 3.3839616005724067,
+      "tokens_seen": 2625044480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1309581126624944e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.3839483667942813,
+      "tokens_seen": 2625175552
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1301556732466704e-05,
+      "loss": 1.312,
+      "theoretical_loss": 3.383935133861886,
+      "tokens_seen": 2625306624
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1293532338308464e-05,
+      "loss": 1.2849,
+      "theoretical_loss": 3.383921901775124,
+      "tokens_seen": 2625437696
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1285507944150224e-05,
+      "loss": 1.336,
+      "theoretical_loss": 3.3839086705338985,
+      "tokens_seen": 2625568768
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.127748354999198e-05,
+      "loss": 1.3166,
+      "theoretical_loss": 3.3838954401381143,
+      "tokens_seen": 2625699840
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1269459155833737e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3838822105876742,
+      "tokens_seen": 2625830912
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1261434761675496e-05,
+      "loss": 1.3289,
+      "theoretical_loss": 3.3838689818824825,
+      "tokens_seen": 2625961984
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1253410367517256e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.383855754022443,
+      "tokens_seen": 2626093056
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1245385973359016e-05,
+      "loss": 1.1983,
+      "theoretical_loss": 3.3838425270074604,
+      "tokens_seen": 2626224128
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.123736157920077e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.3838293008374367,
+      "tokens_seen": 2626355200
+    },
+    {
+      "epoch": 0.59,
+      "objective/train/docs_used": 881674,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1983146667480469,
+      "objective/train/theoretical_loss": 3.3838160755122773,
+      "objective/train/tokens_used": 996945376,
+      "theoretical_loss": 3.3838160755122773,
+      "tokens_seen": 2626486272
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.122933718504253e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3838160755122773,
+      "tokens_seen": 2626486272
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.122131279088429e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.383802851031885,
+      "tokens_seen": 2626617344
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.121328839672605e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3837896273961645,
+      "tokens_seen": 2626748416
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.120526400256781e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.3837764046050194,
+      "tokens_seen": 2626879488
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.119723960840957e-05,
+      "loss": 1.358,
+      "theoretical_loss": 3.3837631826583534,
+      "tokens_seen": 2627010560
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.118921521425132e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.383749961556071,
+      "tokens_seen": 2627141632
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.118119082009308e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.3837367412980757,
+      "tokens_seen": 2627272704
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.117316642593484e-05,
+      "loss": 1.2873,
+      "theoretical_loss": 3.383723521884271,
+      "tokens_seen": 2627403776
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.11651420317766e-05,
+      "loss": 1.2611,
+      "theoretical_loss": 3.383710303314562,
+      "tokens_seen": 2627534848
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.115711763761836e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.383697085588852,
+      "tokens_seen": 2627665920
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.114909324346012e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.3836838687070454,
+      "tokens_seen": 2627796992
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1141068849301875e-05,
+      "loss": 1.3059,
+      "theoretical_loss": 3.3836706526690463,
+      "tokens_seen": 2627928064
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1133044455143635e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.3836574374747577,
+      "tokens_seen": 2628059136
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1125020060985395e-05,
+      "loss": 1.3323,
+      "theoretical_loss": 3.383644223124085,
+      "tokens_seen": 2628190208
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1116995666827154e-05,
+      "loss": 1.2983,
+      "theoretical_loss": 3.383631009616931,
+      "tokens_seen": 2628321280
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1108971272668914e-05,
+      "loss": 1.2523,
+      "theoretical_loss": 3.3836177969532013,
+      "tokens_seen": 2628452352
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.1100946878510674e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.3836045851327987,
+      "tokens_seen": 2628583424
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.109292248435243e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.383591374155628,
+      "tokens_seen": 2628714496
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.108489809019419e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.3835781640215927,
+      "tokens_seen": 2628845568
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.107687369603595e-05,
+      "loss": 1.1753,
+      "theoretical_loss": 3.383564954730598,
+      "tokens_seen": 2628976640
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.106884930187771e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.383551746282547,
+      "tokens_seen": 2629107712
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.106082490771947e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.383538538677345,
+      "tokens_seen": 2629238784
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.105280051356123e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.3835253319148952,
+      "tokens_seen": 2629369856
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.104477611940299e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.383512125995102,
+      "tokens_seen": 2629500928
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.103675172524475e-05,
+      "loss": 1.3016,
+      "theoretical_loss": 3.3834989209178703,
+      "tokens_seen": 2629632000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.10287273310865e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3834857166831034,
+      "tokens_seen": 2629763072
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.102070293692826e-05,
+      "loss": 1.3266,
+      "theoretical_loss": 3.383472513290706,
+      "tokens_seen": 2629894144
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.101267854277002e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.383459310740583,
+      "tokens_seen": 2630025216
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.100465414861178e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.3834461090326378,
+      "tokens_seen": 2630156288
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.099662975445354e-05,
+      "loss": 1.3099,
+      "theoretical_loss": 3.383432908166775,
+      "tokens_seen": 2630287360
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.09886053602953e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.383419708142899,
+      "tokens_seen": 2630418432
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.098058096613706e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.383406508960914,
+      "tokens_seen": 2630549504
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.097255657197881e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3833933106207246,
+      "tokens_seen": 2630680576
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.096453217782057e-05,
+      "loss": 1.2905,
+      "theoretical_loss": 3.383380113122235,
+      "tokens_seen": 2630811648
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.095650778366233e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.38336691646535,
+      "tokens_seen": 2630942720
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.094848338950409e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.383353720649973,
+      "tokens_seen": 2631073792
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.094045899534585e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.38334052567601,
+      "tokens_seen": 2631204864
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.093243460118761e-05,
+      "loss": 1.2999,
+      "theoretical_loss": 3.3833273315433634,
+      "tokens_seen": 2631335936
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.092441020702937e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.383314138251939,
+      "tokens_seen": 2631467008
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 4.0916385812871125e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.3833009458016416,
+      "tokens_seen": 2631598080
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0908361418712885e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.383287754192375,
+      "tokens_seen": 2631729152
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0900337024554645e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3832745634240435,
+      "tokens_seen": 2631860224
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0892312630396405e-05,
+      "loss": 1.302,
+      "theoretical_loss": 3.383261373496552,
+      "tokens_seen": 2631991296
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0884288236238165e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.383248184409805,
+      "tokens_seen": 2632122368
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0876263842079925e-05,
+      "loss": 1.3582,
+      "theoretical_loss": 3.383234996163707,
+      "tokens_seen": 2632253440
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0868239447921684e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.3832218087581625,
+      "tokens_seen": 2632384512
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0860215053763444e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3832086221930764,
+      "tokens_seen": 2632515584
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.08521906596052e-05,
+      "loss": 1.2661,
+      "theoretical_loss": 3.3831954364683527,
+      "tokens_seen": 2632646656
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.084416626544696e-05,
+      "loss": 1.2921,
+      "theoretical_loss": 3.3831822515838965,
+      "tokens_seen": 2632777728
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.083614187128872e-05,
+      "loss": 1.2688,
+      "theoretical_loss": 3.383169067539612,
+      "tokens_seen": 2632908800
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 883708,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2063689231872559,
+      "objective/train/theoretical_loss": 3.383155884335405,
+      "objective/train/tokens_used": 1003498976,
+      "theoretical_loss": 3.383155884335405,
+      "tokens_seen": 2633039872
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.082811747713048e-05,
+      "loss": 1.189,
+      "theoretical_loss": 3.383155884335405,
+      "tokens_seen": 2633039872
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.082009308297224e-05,
+      "loss": 1.3016,
+      "theoretical_loss": 3.3831427019711784,
+      "tokens_seen": 2633170944
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0812068688814e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.383129520446838,
+      "tokens_seen": 2633302016
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.080404429465576e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3831163397622883,
+      "tokens_seen": 2633433088
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.079601990049751e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3831031599174337,
+      "tokens_seen": 2633564160
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.078799550633927e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.3830899809121795,
+      "tokens_seen": 2633695232
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.077997111218103e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.38307680274643,
+      "tokens_seen": 2633826304
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.077194671802279e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.38306362542009,
+      "tokens_seen": 2633957376
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.076392232386455e-05,
+      "loss": 1.1038,
+      "theoretical_loss": 3.383050448933064,
+      "tokens_seen": 2634088448
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.075589792970631e-05,
+      "loss": 1.2111,
+      "theoretical_loss": 3.3830372732852574,
+      "tokens_seen": 2634219520
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.074787353554807e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3830240984765747,
+      "tokens_seen": 2634350592
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.073984914138983e-05,
+      "loss": 1.1663,
+      "theoretical_loss": 3.3830109245069204,
+      "tokens_seen": 2634481664
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.073182474723158e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.3829977513762,
+      "tokens_seen": 2634612736
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.072380035307334e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.382984579084318,
+      "tokens_seen": 2634743808
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.07157759589151e-05,
+      "loss": 1.2628,
+      "theoretical_loss": 3.382971407631179,
+      "tokens_seen": 2634874880
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.070775156475686e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.382958237016688,
+      "tokens_seen": 2635005952
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.069972717059862e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.38294506724075,
+      "tokens_seen": 2635137024
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.069170277644038e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.38293189830327,
+      "tokens_seen": 2635268096
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.068367838228214e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.382918730204153,
+      "tokens_seen": 2635399168
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0675653988123895e-05,
+      "loss": 1.3037,
+      "theoretical_loss": 3.3829055629433036,
+      "tokens_seen": 2635530240
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0667629593965655e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.3828923965206266,
+      "tokens_seen": 2635661312
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0659605199807415e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.3828792309360276,
+      "tokens_seen": 2635792384
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0651580805649175e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.382866066189411,
+      "tokens_seen": 2635923456
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0643556411490935e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.3828529022806824,
+      "tokens_seen": 2636054528
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0635532017332695e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.382839739209746,
+      "tokens_seen": 2636185600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0627507623174455e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3828265769765076,
+      "tokens_seen": 2636316672
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.061948322901621e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.3828134155808716,
+      "tokens_seen": 2636447744
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.061145883485797e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.382800255022744,
+      "tokens_seen": 2636578816
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.060343444069973e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.382787095302029,
+      "tokens_seen": 2636709888
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.059541004654149e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3827739364186318,
+      "tokens_seen": 2636840960
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.058738565238325e-05,
+      "loss": 1.2928,
+      "theoretical_loss": 3.3827607783724574,
+      "tokens_seen": 2636972032
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.057936125822501e-05,
+      "loss": 1.3326,
+      "theoretical_loss": 3.3827476211634115,
+      "tokens_seen": 2637103104
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.057133686406677e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.3827344647913993,
+      "tokens_seen": 2637234176
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.056331246990853e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.382721309256325,
+      "tokens_seen": 2637365248
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.055528807575028e-05,
+      "loss": 1.1935,
+      "theoretical_loss": 3.3827081545580944,
+      "tokens_seen": 2637496320
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.054726368159204e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.3826950006966126,
+      "tokens_seen": 2637627392
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.05392392874338e-05,
+      "loss": 1.2787,
+      "theoretical_loss": 3.382681847671785,
+      "tokens_seen": 2637758464
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.053121489327556e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.382668695483517,
+      "tokens_seen": 2637889536
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.052319049911732e-05,
+      "loss": 1.3075,
+      "theoretical_loss": 3.3826555441317128,
+      "tokens_seen": 2638020608
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.051516610495908e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.382642393616279,
+      "tokens_seen": 2638151680
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.050714171080084e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3826292439371195,
+      "tokens_seen": 2638282752
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.049911731664259e-05,
+      "loss": 1.2825,
+      "theoretical_loss": 3.38261609509414,
+      "tokens_seen": 2638413824
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.049109292248435e-05,
+      "loss": 1.3069,
+      "theoretical_loss": 3.382602947087247,
+      "tokens_seen": 2638544896
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.048306852832611e-05,
+      "loss": 1.1656,
+      "theoretical_loss": 3.382589799916344,
+      "tokens_seen": 2638675968
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.047504413416787e-05,
+      "loss": 1.1707,
+      "theoretical_loss": 3.3825766535813377,
+      "tokens_seen": 2638807040
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.046701974000963e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.382563508082133,
+      "tokens_seen": 2638938112
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.045899534585139e-05,
+      "loss": 1.1598,
+      "theoretical_loss": 3.382550363418635,
+      "tokens_seen": 2639069184
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.045097095169315e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3825372195907493,
+      "tokens_seen": 2639200256
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0442946557534905e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.382524076598381,
+      "tokens_seen": 2639331328
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0434922163376665e-05,
+      "loss": 1.2619,
+      "theoretical_loss": 3.382510934441436,
+      "tokens_seen": 2639462400
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 885915,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2350413799285889,
+      "objective/train/theoretical_loss": 3.382497793119819,
+      "objective/train/tokens_used": 1010052576,
+      "theoretical_loss": 3.382497793119819,
+      "tokens_seen": 2639593472
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0426897769218425e-05,
+      "loss": 1.1938,
+      "theoretical_loss": 3.382497793119819,
+      "tokens_seen": 2639593472
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0418873375060185e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.382484652633436,
+      "tokens_seen": 2639724544
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0410848980901945e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.3824715129821925,
+      "tokens_seen": 2639855616
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0402824586743705e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.382458374165994,
+      "tokens_seen": 2639986688
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0394800192585465e-05,
+      "loss": 1.3324,
+      "theoretical_loss": 3.3824452361847452,
+      "tokens_seen": 2640117760
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0386775798427225e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.3824320990383523,
+      "tokens_seen": 2640248832
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.037875140426898e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.382418962726721,
+      "tokens_seen": 2640379904
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.037072701011074e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.382405827249756,
+      "tokens_seen": 2640510976
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.03627026159525e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.3823926926073637,
+      "tokens_seen": 2640642048
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.035467822179426e-05,
+      "loss": 1.154,
+      "theoretical_loss": 3.3823795587994487,
+      "tokens_seen": 2640773120
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.034665382763602e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.382366425825918,
+      "tokens_seen": 2640904192
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.033862943347778e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.3823532936866756,
+      "tokens_seen": 2641035264
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.033060503931954e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.382340162381628,
+      "tokens_seen": 2641166336
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.032258064516129e-05,
+      "loss": 1.322,
+      "theoretical_loss": 3.3823270319106804,
+      "tokens_seen": 2641297408
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.031455625100305e-05,
+      "loss": 1.2955,
+      "theoretical_loss": 3.382313902273739,
+      "tokens_seen": 2641428480
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.030653185684481e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.382300773470709,
+      "tokens_seen": 2641559552
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.029850746268657e-05,
+      "loss": 1.3463,
+      "theoretical_loss": 3.382287645501496,
+      "tokens_seen": 2641690624
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.029048306852833e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.3822745183660063,
+      "tokens_seen": 2641821696
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.028245867437009e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3822613920641444,
+      "tokens_seen": 2641952768
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.027443428021185e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.3822482665958176,
+      "tokens_seen": 2642083840
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.026640988605361e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3822351419609302,
+      "tokens_seen": 2642214912
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.025838549189536e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3822220181593887,
+      "tokens_seen": 2642345984
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.025036109773712e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.3822088951910985,
+      "tokens_seen": 2642477056
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.024233670357888e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.3821957730559657,
+      "tokens_seen": 2642608128
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.023431230942064e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3821826517538955,
+      "tokens_seen": 2642739200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.02262879152624e-05,
+      "loss": 1.3128,
+      "theoretical_loss": 3.382169531284795,
+      "tokens_seen": 2642870272
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.021826352110416e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.3821564116485683,
+      "tokens_seen": 2643001344
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.021023912694592e-05,
+      "loss": 1.1554,
+      "theoretical_loss": 3.3821432928451225,
+      "tokens_seen": 2643132416
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0202214732787675e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3821301748743626,
+      "tokens_seen": 2643263488
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0194190338629435e-05,
+      "loss": 1.2802,
+      "theoretical_loss": 3.382117057736195,
+      "tokens_seen": 2643394560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0186165944471195e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.3821039414305254,
+      "tokens_seen": 2643525632
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0178141550312955e-05,
+      "loss": 1.1699,
+      "theoretical_loss": 3.38209082595726,
+      "tokens_seen": 2643656704
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0170117156154715e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.3820777113163043,
+      "tokens_seen": 2643787776
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0162092761996475e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3820645975075645,
+      "tokens_seen": 2643918848
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.0154068367838235e-05,
+      "loss": 1.3212,
+      "theoretical_loss": 3.382051484530946,
+      "tokens_seen": 2644049920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.014604397367999e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.382038372386355,
+      "tokens_seen": 2644180992
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.013801957952175e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.3820252610736983,
+      "tokens_seen": 2644312064
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.012999518536351e-05,
+      "loss": 1.3329,
+      "theoretical_loss": 3.382012150592881,
+      "tokens_seen": 2644443136
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.012197079120527e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.3819990409438088,
+      "tokens_seen": 2644574208
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.011394639704703e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3819859321263888,
+      "tokens_seen": 2644705280
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.010592200288879e-05,
+      "loss": 1.277,
+      "theoretical_loss": 3.381972824140526,
+      "tokens_seen": 2644836352
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.009789760873055e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.381959716986127,
+      "tokens_seen": 2644967424
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.008987321457231e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.381946610663098,
+      "tokens_seen": 2645098496
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.008184882041406e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.3819335051713444,
+      "tokens_seen": 2645229568
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.007382442625582e-05,
+      "loss": 1.3061,
+      "theoretical_loss": 3.381920400510773,
+      "tokens_seen": 2645360640
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.006580003209758e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3819072966812893,
+      "tokens_seen": 2645491712
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.005777563793934e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.3818941936827995,
+      "tokens_seen": 2645622784
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.00497512437811e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.381881091515211,
+      "tokens_seen": 2645753856
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.004172684962286e-05,
+      "loss": 1.2327,
+      "theoretical_loss": 3.381867990178428,
+      "tokens_seen": 2645884928
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.003370245546462e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3818548896723577,
+      "tokens_seen": 2646016000
+    },
+    {
+      "epoch": 0.6,
+      "objective/train/docs_used": 888056,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2244160175323486,
+      "objective/train/theoretical_loss": 3.3818417899969067,
+      "objective/train/tokens_used": 1016606176,
+      "theoretical_loss": 3.3818417899969067,
+      "tokens_seen": 2646147072
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.002567806130637e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.3818417899969067,
+      "tokens_seen": 2646147072
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.001765366714813e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.3818286911519797,
+      "tokens_seen": 2646278144
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.000962927298989e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3818155931374845,
+      "tokens_seen": 2646409216
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.000160487883165e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.381802495953327,
+      "tokens_seen": 2646540288
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.999358048467341e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.3817893995994126,
+      "tokens_seen": 2646671360
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.998555609051517e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.381776304075648,
+      "tokens_seen": 2646802432
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.997753169635693e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.38176320938194,
+      "tokens_seen": 2646933504
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9969507302198685e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.3817501155181944,
+      "tokens_seen": 2647064576
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9961482908040445e-05,
+      "loss": 1.3391,
+      "theoretical_loss": 3.3817370224843173,
+      "tokens_seen": 2647195648
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9953458513882205e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.3817239302802156,
+      "tokens_seen": 2647326720
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9945434119723965e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.3817108389057946,
+      "tokens_seen": 2647457792
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9937409725565725e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.381697748360962,
+      "tokens_seen": 2647588864
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9929385331407485e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3816846586456233,
+      "tokens_seen": 2647719936
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.992136093724924e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.3816715697596855,
+      "tokens_seen": 2647851008
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.9913336543091e-05,
+      "loss": 1.3189,
+      "theoretical_loss": 3.381658481703054,
+      "tokens_seen": 2647982080
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.990531214893276e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.3816453944756364,
+      "tokens_seen": 2648113152
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.989728775477452e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.381632308077338,
+      "tokens_seen": 2648244224
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.988926336061628e-05,
+      "loss": 1.301,
+      "theoretical_loss": 3.381619222508066,
+      "tokens_seen": 2648375296
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.988123896645804e-05,
+      "loss": 1.3272,
+      "theoretical_loss": 3.3816061377677267,
+      "tokens_seen": 2648506368
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.987321457229979e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.381593053856226,
+      "tokens_seen": 2648637440
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.986519017814155e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3815799707734717,
+      "tokens_seen": 2648768512
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.985716578398331e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.381566888519369,
+      "tokens_seen": 2648899584
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.984914138982507e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.381553807093825,
+      "tokens_seen": 2649030656
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.984111699566683e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.381540726496746,
+      "tokens_seen": 2649161728
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.983309260150859e-05,
+      "loss": 1.3143,
+      "theoretical_loss": 3.381527646728039,
+      "tokens_seen": 2649292800
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9825068207350343e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.38151456778761,
+      "tokens_seen": 2649423872
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.98170438131921e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.3815014896753657,
+      "tokens_seen": 2649554944
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.980901941903386e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.381488412391213,
+      "tokens_seen": 2649686016
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.980099502487562e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.3814753359350584,
+      "tokens_seen": 2649817088
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.979297063071738e-05,
+      "loss": 1.2957,
+      "theoretical_loss": 3.381462260306808,
+      "tokens_seen": 2649948160
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.978494623655914e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3814491855063693,
+      "tokens_seen": 2650079232
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9776921842400896e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.381436111533648,
+      "tokens_seen": 2650210304
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9768897448242656e-05,
+      "loss": 1.279,
+      "theoretical_loss": 3.3814230383885517,
+      "tokens_seen": 2650341376
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9760873054084416e-05,
+      "loss": 1.3185,
+      "theoretical_loss": 3.381409966070987,
+      "tokens_seen": 2650472448
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9752848659926176e-05,
+      "loss": 1.2656,
+      "theoretical_loss": 3.3813968945808597,
+      "tokens_seen": 2650603520
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9744824265767936e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.3813838239180773,
+      "tokens_seen": 2650734592
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9736799871609696e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.381370754082546,
+      "tokens_seen": 2650865664
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.972877547745145e-05,
+      "loss": 1.1699,
+      "theoretical_loss": 3.3813576850741733,
+      "tokens_seen": 2650996736
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.972075108329321e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.3813446168928656,
+      "tokens_seen": 2651127808
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.971272668913497e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.381331549538529,
+      "tokens_seen": 2651258880
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.970470229497673e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.3813184830110714,
+      "tokens_seen": 2651389952
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.969667790081849e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.381305417310399,
+      "tokens_seen": 2651521024
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.968865350666025e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.381292352436419,
+      "tokens_seen": 2651652096
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.968062911250201e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3812792883890372,
+      "tokens_seen": 2651783168
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.967260471834376e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.3812662251681616,
+      "tokens_seen": 2651914240
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.966458032418552e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3812531627736986,
+      "tokens_seen": 2652045312
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.965655593002728e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.381240101205555,
+      "tokens_seen": 2652176384
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.964853153586904e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3812270404636378,
+      "tokens_seen": 2652307456
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.96405071417108e-05,
+      "loss": 1.3092,
+      "theoretical_loss": 3.381213980547854,
+      "tokens_seen": 2652438528
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.963248274755256e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3812009214581105,
+      "tokens_seen": 2652569600
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 890384,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2147217988967896,
+      "objective/train/theoretical_loss": 3.381187863194314,
+      "objective/train/tokens_used": 1023159776,
+      "theoretical_loss": 3.381187863194314,
+      "tokens_seen": 2652700672
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.962445835339432e-05,
+      "loss": 1.2263,
+      "theoretical_loss": 3.381187863194314,
+      "tokens_seen": 2652700672
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.961643395923608e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.381174805756372,
+      "tokens_seen": 2652831744
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9608409565077834e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.3811617491441908,
+      "tokens_seen": 2652962816
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9600385170919594e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.381148693357678,
+      "tokens_seen": 2653093888
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9592360776761354e-05,
+      "loss": 1.2907,
+      "theoretical_loss": 3.38113563839674,
+      "tokens_seen": 2653224960
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9584336382603114e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.381122584261284,
+      "tokens_seen": 2653356032
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9576311988444873e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.3811095309512176,
+      "tokens_seen": 2653487104
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.956828759428663e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.3810964784664472,
+      "tokens_seen": 2653618176
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.956026320012839e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.38108342680688,
+      "tokens_seen": 2653749248
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9552238805970146e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.381070375972423,
+      "tokens_seen": 2653880320
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9544214411811906e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.381057325962984,
+      "tokens_seen": 2654011392
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9536190017653666e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.3810442767784687,
+      "tokens_seen": 2654142464
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9528165623495426e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3810312284187853,
+      "tokens_seen": 2654273536
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9520141229337186e-05,
+      "loss": 1.2964,
+      "theoretical_loss": 3.3810181808838413,
+      "tokens_seen": 2654404608
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9512116835178946e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.3810051341735425,
+      "tokens_seen": 2654535680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9504092441020706e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.380992088287797,
+      "tokens_seen": 2654666752
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.949606804686246e-05,
+      "loss": 1.2184,
+      "theoretical_loss": 3.3809790432265117,
+      "tokens_seen": 2654797824
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.948804365270422e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.380965998989594,
+      "tokens_seen": 2654928896
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.948001925854598e-05,
+      "loss": 1.2807,
+      "theoretical_loss": 3.380952955576951,
+      "tokens_seen": 2655059968
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.947199486438774e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.38093991298849,
+      "tokens_seen": 2655191040
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.94639704702295e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.380926871224118,
+      "tokens_seen": 2655322112
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.945594607607126e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.380913830283742,
+      "tokens_seen": 2655453184
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.944792168191302e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.3809007901672703,
+      "tokens_seen": 2655584256
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.943989728775478e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.380887750874609,
+      "tokens_seen": 2655715328
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.943187289359653e-05,
+      "loss": 1.1885,
+      "theoretical_loss": 3.3808747124056664,
+      "tokens_seen": 2655846400
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.942384849943829e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.3808616747603493,
+      "tokens_seen": 2655977472
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.941582410528005e-05,
+      "loss": 1.1811,
+      "theoretical_loss": 3.380848637938565,
+      "tokens_seen": 2656108544
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.940779971112181e-05,
+      "loss": 1.2888,
+      "theoretical_loss": 3.380835601940221,
+      "tokens_seen": 2656239616
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.939977531696357e-05,
+      "loss": 1.3113,
+      "theoretical_loss": 3.3808225667652243,
+      "tokens_seen": 2656370688
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.939175092280533e-05,
+      "loss": 1.2979,
+      "theoretical_loss": 3.380809532413483,
+      "tokens_seen": 2656501760
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.938372652864709e-05,
+      "loss": 1.2128,
+      "theoretical_loss": 3.380796498884904,
+      "tokens_seen": 2656632832
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9375702134488844e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.3807834661793947,
+      "tokens_seen": 2656763904
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9367677740330604e-05,
+      "loss": 1.1681,
+      "theoretical_loss": 3.3807704342968625,
+      "tokens_seen": 2656894976
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9359653346172364e-05,
+      "loss": 1.1728,
+      "theoretical_loss": 3.380757403237215,
+      "tokens_seen": 2657026048
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9351628952014124e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.38074437300036,
+      "tokens_seen": 2657157120
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9343604557855884e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.380731343586204,
+      "tokens_seen": 2657288192
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9335580163697644e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.380718314994655,
+      "tokens_seen": 2657419264
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9327555769539403e-05,
+      "loss": 1.2004,
+      "theoretical_loss": 3.380705287225621,
+      "tokens_seen": 2657550336
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.931953137538116e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.380692260279009,
+      "tokens_seen": 2657681408
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9311506981222916e-05,
+      "loss": 1.1618,
+      "theoretical_loss": 3.380679234154726,
+      "tokens_seen": 2657812480
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9303482587064676e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.3806662088526807,
+      "tokens_seen": 2657943552
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9295458192906436e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.38065318437278,
+      "tokens_seen": 2658074624
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9287433798748196e-05,
+      "loss": 1.217,
+      "theoretical_loss": 3.380640160714931,
+      "tokens_seen": 2658205696
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9279409404589956e-05,
+      "loss": 1.1342,
+      "theoretical_loss": 3.380627137879042,
+      "tokens_seen": 2658336768
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9271385010431716e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.3806141158650207,
+      "tokens_seen": 2658467840
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9263360616273476e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.380601094672774,
+      "tokens_seen": 2658598912
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.925533622211523e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3805880743022105,
+      "tokens_seen": 2658729984
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.924731182795699e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.380575054753237,
+      "tokens_seen": 2658861056
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.923928743379875e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.3805620360257613,
+      "tokens_seen": 2658992128
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.923126303964051e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3805490181196918,
+      "tokens_seen": 2659123200
+    },
+    {
+      "epoch": 0.61,
+      "objective/train/docs_used": 892248,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1734418869018555,
+      "objective/train/theoretical_loss": 3.380536001034935,
+      "objective/train/tokens_used": 1029713376,
+      "theoretical_loss": 3.380536001034935,
+      "tokens_seen": 2659254272
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.922323864548227e-05,
+      "loss": 1.1396,
+      "theoretical_loss": 3.380536001034935,
+      "tokens_seen": 2659254272
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.921521425132403e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.3805229847713996,
+      "tokens_seen": 2659385344
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.920718985716579e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.380509969328993,
+      "tokens_seen": 2659516416
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.919916546300754e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.3804969547076227,
+      "tokens_seen": 2659647488
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.91911410688493e-05,
+      "loss": 1.1449,
+      "theoretical_loss": 3.3804839409071965,
+      "tokens_seen": 2659778560
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.918311667469106e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3804709279276226,
+      "tokens_seen": 2659909632
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.917509228053282e-05,
+      "loss": 1.3003,
+      "theoretical_loss": 3.3804579157688086,
+      "tokens_seen": 2660040704
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.916706788637458e-05,
+      "loss": 1.2037,
+      "theoretical_loss": 3.3804449044306617,
+      "tokens_seen": 2660171776
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.915904349221634e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3804318939130904,
+      "tokens_seen": 2660302848
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.91510190980581e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3804188842160023,
+      "tokens_seen": 2660433920
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.914299470389986e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3804058753393056,
+      "tokens_seen": 2660564992
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9134970309741614e-05,
+      "loss": 1.0909,
+      "theoretical_loss": 3.3803928672829073,
+      "tokens_seen": 2660696064
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9126945915583374e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.380379860046716,
+      "tokens_seen": 2660827136
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9118921521425134e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.380366853630639,
+      "tokens_seen": 2660958208
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9110897127266894e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.380353848034585,
+      "tokens_seen": 2661089280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9102872733108654e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.380340843258461,
+      "tokens_seen": 2661220352
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9094848338950414e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.380327839302176,
+      "tokens_seen": 2661351424
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9086823944792174e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.380314836165637,
+      "tokens_seen": 2661482496
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.907879955063393e-05,
+      "loss": 1.1705,
+      "theoretical_loss": 3.3803018338487525,
+      "tokens_seen": 2661613568
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9070775156475687e-05,
+      "loss": 1.1789,
+      "theoretical_loss": 3.3802888323514297,
+      "tokens_seen": 2661744640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9062750762317446e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.3802758316735777,
+      "tokens_seen": 2661875712
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9054726368159206e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.380262831815104,
+      "tokens_seen": 2662006784
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9046701974000966e-05,
+      "loss": 1.1269,
+      "theoretical_loss": 3.380249832775916,
+      "tokens_seen": 2662137856
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9038677579842726e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.380236834555923,
+      "tokens_seen": 2662268928
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9030653185684486e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.3802238371550315,
+      "tokens_seen": 2662400000
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.902262879152624e-05,
+      "loss": 1.1427,
+      "theoretical_loss": 3.3802108405731515,
+      "tokens_seen": 2662531072
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.9014604397368e-05,
+      "loss": 1.181,
+      "theoretical_loss": 3.3801978448101893,
+      "tokens_seen": 2662662144
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.900658000320976e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.3801848498660534,
+      "tokens_seen": 2662793216
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.899855560905152e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.380171855740653,
+      "tokens_seen": 2662924288
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.899053121489328e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3801588624338947,
+      "tokens_seen": 2663055360
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.898250682073504e-05,
+      "loss": 1.1465,
+      "theoretical_loss": 3.3801458699456877,
+      "tokens_seen": 2663186432
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.89744824265768e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.3801328782759397,
+      "tokens_seen": 2663317504
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.896645803241856e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.380119887424559,
+      "tokens_seen": 2663448576
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.895843363826031e-05,
+      "loss": 1.1868,
+      "theoretical_loss": 3.3801068973914536,
+      "tokens_seen": 2663579648
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.895040924410207e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.3800939081765318,
+      "tokens_seen": 2663710720
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.894238484994383e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.380080919779702,
+      "tokens_seen": 2663841792
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.893436045578559e-05,
+      "loss": 1.1769,
+      "theoretical_loss": 3.3800679322008724,
+      "tokens_seen": 2663972864
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.892633606162735e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.380054945439951,
+      "tokens_seen": 2664103936
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.891831166746911e-05,
+      "loss": 1.1598,
+      "theoretical_loss": 3.3800419594968463,
+      "tokens_seen": 2664235008
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.891028727331087e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.380028974371466,
+      "tokens_seen": 2664366080
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.8902262879152624e-05,
+      "loss": 1.2039,
+      "theoretical_loss": 3.3800159900637192,
+      "tokens_seen": 2664497152
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.8894238484994384e-05,
+      "loss": 1.1667,
+      "theoretical_loss": 3.3800030065735136,
+      "tokens_seen": 2664628224
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8886214090836144e-05,
+      "loss": 1.1723,
+      "theoretical_loss": 3.379990023900758,
+      "tokens_seen": 2664759296
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8878189696677904e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.3799770420453603,
+      "tokens_seen": 2664890368
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8870165302519664e-05,
+      "loss": 1.1567,
+      "theoretical_loss": 3.379964061007229,
+      "tokens_seen": 2665021440
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8862140908361424e-05,
+      "loss": 1.1076,
+      "theoretical_loss": 3.379951080786272,
+      "tokens_seen": 2665152512
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8854116514203184e-05,
+      "loss": 1.1113,
+      "theoretical_loss": 3.3799381013823986,
+      "tokens_seen": 2665283584
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.884609212004494e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.379925122795517,
+      "tokens_seen": 2665414656
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.88380677258867e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.379912145025535,
+      "tokens_seen": 2665545728
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.883004333172846e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.3798991680723613,
+      "tokens_seen": 2665676800
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 894393,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.298959732055664,
+      "objective/train/theoretical_loss": 3.3798861919359044,
+      "objective/train/tokens_used": 1036266976,
+      "theoretical_loss": 3.3798861919359044,
+      "tokens_seen": 2665807872
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8822018937570217e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.3798861919359044,
+      "tokens_seen": 2665807872
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8813994543411976e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.379873216616073,
+      "tokens_seen": 2665938944
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8805970149253736e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.379860242112775,
+      "tokens_seen": 2666070016
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8797945755095496e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.379847268425919,
+      "tokens_seen": 2666201088
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8789921360937256e-05,
+      "loss": 1.1532,
+      "theoretical_loss": 3.379834295555414,
+      "tokens_seen": 2666332160
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.878189696677901e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.379821323501168,
+      "tokens_seen": 2666463232
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.877387257262077e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.3798083522630895,
+      "tokens_seen": 2666594304
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.876584817846253e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3797953818410877,
+      "tokens_seen": 2666725376
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.875782378430429e-05,
+      "loss": 1.1562,
+      "theoretical_loss": 3.37978241223507,
+      "tokens_seen": 2666856448
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.874979939014605e-05,
+      "loss": 1.148,
+      "theoretical_loss": 3.379769443444946,
+      "tokens_seen": 2666987520
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.874177499598781e-05,
+      "loss": 1.1903,
+      "theoretical_loss": 3.379756475470624,
+      "tokens_seen": 2667118592
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.873375060182957e-05,
+      "loss": 1.1462,
+      "theoretical_loss": 3.3797435083120124,
+      "tokens_seen": 2667249664
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.872572620767132e-05,
+      "loss": 1.1646,
+      "theoretical_loss": 3.3797305419690202,
+      "tokens_seen": 2667380736
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.871770181351308e-05,
+      "loss": 1.1749,
+      "theoretical_loss": 3.3797175764415552,
+      "tokens_seen": 2667511808
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.870967741935484e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.379704611729527,
+      "tokens_seen": 2667642880
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.87016530251966e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3796916478328436,
+      "tokens_seen": 2667773952
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.869362863103836e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.379678684751414,
+      "tokens_seen": 2667905024
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.868560423688012e-05,
+      "loss": 1.1516,
+      "theoretical_loss": 3.3796657224851465,
+      "tokens_seen": 2668036096
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.867757984272188e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.37965276103395,
+      "tokens_seen": 2668167168
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.866955544856364e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.379639800397734,
+      "tokens_seen": 2668298240
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8661531054405394e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.379626840576406,
+      "tokens_seen": 2668429312
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8653506660247154e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.379613881569875,
+      "tokens_seen": 2668560384
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8645482266088914e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3796009233780504,
+      "tokens_seen": 2668691456
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8637457871930674e-05,
+      "loss": 1.1363,
+      "theoretical_loss": 3.3795879660008405,
+      "tokens_seen": 2668822528
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8629433477772434e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.3795750094381543,
+      "tokens_seen": 2668953600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8621409083614194e-05,
+      "loss": 1.1447,
+      "theoretical_loss": 3.3795620536899,
+      "tokens_seen": 2669084672
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.861338468945595e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3795490987559873,
+      "tokens_seen": 2669215744
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.860536029529771e-05,
+      "loss": 1.1972,
+      "theoretical_loss": 3.3795361446363246,
+      "tokens_seen": 2669346816
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.859733590113947e-05,
+      "loss": 1.1842,
+      "theoretical_loss": 3.3795231913308204,
+      "tokens_seen": 2669477888
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.858931150698123e-05,
+      "loss": 1.1588,
+      "theoretical_loss": 3.3795102388393845,
+      "tokens_seen": 2669608960
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8581287112822987e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.3794972871619247,
+      "tokens_seen": 2669740032
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8573262718664747e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3794843362983507,
+      "tokens_seen": 2669871104
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.85652383245065e-05,
+      "loss": 1.1629,
+      "theoretical_loss": 3.3794713862485706,
+      "tokens_seen": 2670002176
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.855721393034826e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3794584370124943,
+      "tokens_seen": 2670133248
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.854918953619002e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.37944548859003,
+      "tokens_seen": 2670264320
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.854116514203178e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3794325409810866,
+      "tokens_seen": 2670395392
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.853314074787354e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.3794195941855736,
+      "tokens_seen": 2670526464
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.85251163537153e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3794066482033998,
+      "tokens_seen": 2670657536
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.851709195955705e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.379393703034474,
+      "tokens_seen": 2670788608
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.850906756539881e-05,
+      "loss": 1.1879,
+      "theoretical_loss": 3.3793807586787055,
+      "tokens_seen": 2670919680
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.850104317124057e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.379367815136003,
+      "tokens_seen": 2671050752
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.849301877708233e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.3793548724062754,
+      "tokens_seen": 2671181824
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.848499438292409e-05,
+      "loss": 1.1655,
+      "theoretical_loss": 3.3793419304894323,
+      "tokens_seen": 2671312896
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.847696998876585e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3793289893853826,
+      "tokens_seen": 2671443968
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8468945594607605e-05,
+      "loss": 1.1797,
+      "theoretical_loss": 3.379316049094035,
+      "tokens_seen": 2671575040
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8460921200449365e-05,
+      "loss": 1.1907,
+      "theoretical_loss": 3.379303109615299,
+      "tokens_seen": 2671706112
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8452896806291125e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3792901709490835,
+      "tokens_seen": 2671837184
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8444872412132885e-05,
+      "loss": 1.1628,
+      "theoretical_loss": 3.3792772330952974,
+      "tokens_seen": 2671968256
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8436848017974645e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.3792642960538504,
+      "tokens_seen": 2672099328
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8428823623816404e-05,
+      "loss": 1.1829,
+      "theoretical_loss": 3.3792513598246514,
+      "tokens_seen": 2672230400
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 896553,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2324345111846924,
+      "objective/train/theoretical_loss": 3.3792384244076095,
+      "objective/train/tokens_used": 1042820576,
+      "theoretical_loss": 3.3792384244076095,
+      "tokens_seen": 2672361472
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.842079922965816e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3792384244076095,
+      "tokens_seen": 2672361472
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.841277483549992e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.379225489802634,
+      "tokens_seen": 2672492544
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.840475044134168e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3792125560096338,
+      "tokens_seen": 2672623616
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.839672604718344e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.3791996230285184,
+      "tokens_seen": 2672754688
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.83887016530252e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.379186690859197,
+      "tokens_seen": 2672885760
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.838067725886696e-05,
+      "loss": 1.1697,
+      "theoretical_loss": 3.3791737595015787,
+      "tokens_seen": 2673016832
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.837265286470871e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3791608289555732,
+      "tokens_seen": 2673147904
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.836462847055047e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.379147899221089,
+      "tokens_seen": 2673278976
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.835660407639223e-05,
+      "loss": 1.1931,
+      "theoretical_loss": 3.379134970298036,
+      "tokens_seen": 2673410048
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.834857968223399e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.379122042186323,
+      "tokens_seen": 2673541120
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.834055528807575e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.3791091148858596,
+      "tokens_seen": 2673672192
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.833253089391751e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.379096188396556,
+      "tokens_seen": 2673803264
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.832450649975927e-05,
+      "loss": 1.145,
+      "theoretical_loss": 3.3790832627183196,
+      "tokens_seen": 2673934336
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.831648210560103e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.3790703378510614,
+      "tokens_seen": 2674065408
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.830845771144278e-05,
+      "loss": 1.1844,
+      "theoretical_loss": 3.37905741379469,
+      "tokens_seen": 2674196480
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.830043331728454e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.379044490549115,
+      "tokens_seen": 2674327552
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.82924089231263e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.3790315681142458,
+      "tokens_seen": 2674458624
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.828438452896806e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3790186464899916,
+      "tokens_seen": 2674589696
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.827636013480982e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3790057256762625,
+      "tokens_seen": 2674720768
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.826833574065158e-05,
+      "loss": 1.1485,
+      "theoretical_loss": 3.3789928056729672,
+      "tokens_seen": 2674851840
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.826031134649334e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.3789798864800153,
+      "tokens_seen": 2674982912
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8252286952335095e-05,
+      "loss": 1.1909,
+      "theoretical_loss": 3.3789669680973167,
+      "tokens_seen": 2675113984
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8244262558176855e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.37895405052478,
+      "tokens_seen": 2675245056
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8236238164018615e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.378941133762316,
+      "tokens_seen": 2675376128
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8228213769860375e-05,
+      "loss": 1.1492,
+      "theoretical_loss": 3.3789282178098334,
+      "tokens_seen": 2675507200
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8220189375702135e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.3789153026672416,
+      "tokens_seen": 2675638272
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8212164981543895e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3789023883344504,
+      "tokens_seen": 2675769344
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8204140587385655e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3788894748113694,
+      "tokens_seen": 2675900416
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8196116193227415e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.378876562097908,
+      "tokens_seen": 2676031488
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.818809179906917e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.3788636501939764,
+      "tokens_seen": 2676162560
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.818006740491093e-05,
+      "loss": 1.222,
+      "theoretical_loss": 3.3788507390994833,
+      "tokens_seen": 2676293632
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.817204301075269e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.378837828814339,
+      "tokens_seen": 2676424704
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.816401861659445e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.3788249193384523,
+      "tokens_seen": 2676555776
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.815599422243621e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.3788120106717336,
+      "tokens_seen": 2676686848
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.814796982827797e-05,
+      "loss": 1.1489,
+      "theoretical_loss": 3.3787991028140927,
+      "tokens_seen": 2676817920
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.813994543411973e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.378786195765439,
+      "tokens_seen": 2676948992
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.813192103996148e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.378773289525682,
+      "tokens_seen": 2677080064
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.812389664580324e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.3787603840947313,
+      "tokens_seen": 2677211136
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8115872251645e-05,
+      "loss": 1.2766,
+      "theoretical_loss": 3.378747479472497,
+      "tokens_seen": 2677342208
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.810784785748676e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3787345756588887,
+      "tokens_seen": 2677473280
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.809982346332852e-05,
+      "loss": 1.1836,
+      "theoretical_loss": 3.378721672653816,
+      "tokens_seen": 2677604352
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.809179906917028e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.378708770457189,
+      "tokens_seen": 2677735424
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.808377467501204e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3786958690689177,
+      "tokens_seen": 2677866496
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.807575028085379e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.378682968488911,
+      "tokens_seen": 2677997568
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.806772588669555e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3786700687170788,
+      "tokens_seen": 2678128640
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.805970149253731e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.378657169753332,
+      "tokens_seen": 2678259712
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.805167709837907e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3786442715975795,
+      "tokens_seen": 2678390784
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.804365270422083e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.378631374249731,
+      "tokens_seen": 2678521856
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.803562831006259e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.3786184777096975,
+      "tokens_seen": 2678652928
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.802760391590435e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3786055819773875,
+      "tokens_seen": 2678784000
+    },
+    {
+      "epoch": 0.62,
+      "objective/train/docs_used": 898681,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.358594298362732,
+      "objective/train/theoretical_loss": 3.378592687052712,
+      "objective/train/tokens_used": 1049374176,
+      "theoretical_loss": 3.378592687052712,
+      "tokens_seen": 2678915072
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.801957952174611e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.378592687052712,
+      "tokens_seen": 2678915072
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8011555127587865e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.37857979293558,
+      "tokens_seen": 2679046144
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.8003530733429625e-05,
+      "loss": 1.314,
+      "theoretical_loss": 3.3785668996259024,
+      "tokens_seen": 2679177216
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.7995506339271385e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.378554007123588,
+      "tokens_seen": 2679308288
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.7987481945113145e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.378541115428548,
+      "tokens_seen": 2679439360
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.7979457550954905e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.378528224540691,
+      "tokens_seen": 2679570432
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.7971433156796665e-05,
+      "loss": 1.2049,
+      "theoretical_loss": 3.3785153344599284,
+      "tokens_seen": 2679701504
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.7963408762638425e-05,
+      "loss": 1.1701,
+      "theoretical_loss": 3.378502445186169,
+      "tokens_seen": 2679832576
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.795538436848018e-05,
+      "loss": 1.2482,
+      "theoretical_loss": 3.378489556719323,
+      "tokens_seen": 2679963648
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.794735997432194e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3784766690593013,
+      "tokens_seen": 2680094720
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.79393355801637e-05,
+      "loss": 1.2402,
+      "theoretical_loss": 3.3784637822060137,
+      "tokens_seen": 2680225792
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.793131118600546e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.3784508961593693,
+      "tokens_seen": 2680356864
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.792328679184722e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.378438010919279,
+      "tokens_seen": 2680487936
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.791526239768898e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.378425126485652,
+      "tokens_seen": 2680619008
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.790723800353074e-05,
+      "loss": 1.2727,
+      "theoretical_loss": 3.3784122428584,
+      "tokens_seen": 2680750080
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.789921360937249e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.378399360037432,
+      "tokens_seen": 2680881152
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.789118921521425e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.378386478022658,
+      "tokens_seen": 2681012224
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 3.788316482105601e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3783735968139883,
+      "tokens_seen": 2681143296
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.787514042689777e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.3783607164113336,
+      "tokens_seen": 2681274368
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.786711603273953e-05,
+      "loss": 1.1589,
+      "theoretical_loss": 3.378347836814603,
+      "tokens_seen": 2681405440
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.785909163858129e-05,
+      "loss": 1.3247,
+      "theoretical_loss": 3.378334958023708,
+      "tokens_seen": 2681536512
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.785106724442305e-05,
+      "loss": 1.2848,
+      "theoretical_loss": 3.378322080038558,
+      "tokens_seen": 2681667584
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.784304285026481e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.378309202859063,
+      "tokens_seen": 2681798656
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.783501845610656e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.378296326485134,
+      "tokens_seen": 2681929728
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.782699406194832e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3782834509166806,
+      "tokens_seen": 2682060800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.781896966779008e-05,
+      "loss": 1.1731,
+      "theoretical_loss": 3.378270576153613,
+      "tokens_seen": 2682191872
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.781094527363184e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.378257702195842,
+      "tokens_seen": 2682322944
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.78029208794736e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3782448290432776,
+      "tokens_seen": 2682454016
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.779489648531536e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.37823195669583,
+      "tokens_seen": 2682585088
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.778687209115712e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3782190851534097,
+      "tokens_seen": 2682716160
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7778847696998876e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.378206214415927,
+      "tokens_seen": 2682847232
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7770823302840635e-05,
+      "loss": 1.3063,
+      "theoretical_loss": 3.378193344483292,
+      "tokens_seen": 2682978304
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7762798908682395e-05,
+      "loss": 1.3309,
+      "theoretical_loss": 3.3781804753554154,
+      "tokens_seen": 2683109376
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7754774514524155e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.378167607032207,
+      "tokens_seen": 2683240448
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7746750120365915e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3781547395135783,
+      "tokens_seen": 2683371520
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7738725726207675e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.3781418727994383,
+      "tokens_seen": 2683502592
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7730701332049435e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.378129006889699,
+      "tokens_seen": 2683633664
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7722676937891195e-05,
+      "loss": 1.1817,
+      "theoretical_loss": 3.378116141784269,
+      "tokens_seen": 2683764736
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.771465254373295e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.37810327748306,
+      "tokens_seen": 2683895808
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.770662814957471e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.378090413985982,
+      "tokens_seen": 2684026880
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.769860375541647e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.378077551292946,
+      "tokens_seen": 2684157952
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.769057936125823e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.3780646894038617,
+      "tokens_seen": 2684289024
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.768255496709999e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.37805182831864,
+      "tokens_seen": 2684420096
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.767453057294175e-05,
+      "loss": 1.2024,
+      "theoretical_loss": 3.378038968037192,
+      "tokens_seen": 2684551168
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.766650617878351e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.3780261085594265,
+      "tokens_seen": 2684682240
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.765848178462526e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3780132498852558,
+      "tokens_seen": 2684813312
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.765045739046702e-05,
+      "loss": 1.1493,
+      "theoretical_loss": 3.37800039201459,
+      "tokens_seen": 2684944384
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.764243299630878e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.377987534947339,
+      "tokens_seen": 2685075456
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.763440860215054e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.3779746786834144,
+      "tokens_seen": 2685206528
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.76263842079923e-05,
+      "loss": 1.3153,
+      "theoretical_loss": 3.3779618232227255,
+      "tokens_seen": 2685337600
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 900968,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2544584274291992,
+      "objective/train/theoretical_loss": 3.3779489685651845,
+      "objective/train/tokens_used": 1055927776,
+      "theoretical_loss": 3.3779489685651845,
+      "tokens_seen": 2685468672
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.761835981383406e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3779489685651845,
+      "tokens_seen": 2685468672
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.761033541967582e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.3779361147107005,
+      "tokens_seen": 2685599744
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.760231102551757e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.3779232616591854,
+      "tokens_seen": 2685730816
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.759428663135933e-05,
+      "loss": 1.1757,
+      "theoretical_loss": 3.377910409410549,
+      "tokens_seen": 2685861888
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.758626223720109e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.3778975579647024,
+      "tokens_seen": 2685992960
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.757823784304285e-05,
+      "loss": 1.1688,
+      "theoretical_loss": 3.377884707321556,
+      "tokens_seen": 2686124032
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.757021344888461e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.3778718574810207,
+      "tokens_seen": 2686255104
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.756218905472637e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.377859008443007,
+      "tokens_seen": 2686386176
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.755416466056813e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3778461602074263,
+      "tokens_seen": 2686517248
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.754614026640989e-05,
+      "loss": 1.1775,
+      "theoretical_loss": 3.3778333127741886,
+      "tokens_seen": 2686648320
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7538115872251646e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.3778204661432047,
+      "tokens_seen": 2686779392
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7530091478093406e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.3778076203143854,
+      "tokens_seen": 2686910464
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7522067083935165e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3777947752876423,
+      "tokens_seen": 2687041536
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7514042689776925e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.377781931062885,
+      "tokens_seen": 2687172608
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7506018295618685e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.377769087640025,
+      "tokens_seen": 2687303680
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7497993901460445e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.377756245018973,
+      "tokens_seen": 2687434752
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7489969507302205e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.3777434031996396,
+      "tokens_seen": 2687565824
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.748194511314396e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.3777305621819362,
+      "tokens_seen": 2687696896
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.747392071898572e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.377717721965773,
+      "tokens_seen": 2687827968
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.746589632482748e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.3777048825510616,
+      "tokens_seen": 2687959040
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.745787193066924e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3776920439377127,
+      "tokens_seen": 2688090112
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7449847536511e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.377679206125636,
+      "tokens_seen": 2688221184
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.744182314235276e-05,
+      "loss": 1.279,
+      "theoretical_loss": 3.3776663691147446,
+      "tokens_seen": 2688352256
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.743379874819452e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.377653532904948,
+      "tokens_seen": 2688483328
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.742577435403627e-05,
+      "loss": 1.1898,
+      "theoretical_loss": 3.3776406974961573,
+      "tokens_seen": 2688614400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.741774995987803e-05,
+      "loss": 1.2703,
+      "theoretical_loss": 3.377627862888284,
+      "tokens_seen": 2688745472
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.740972556571979e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.3776150290812383,
+      "tokens_seen": 2688876544
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.740170117156155e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.377602196074932,
+      "tokens_seen": 2689007616
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.739367677740331e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.377589363869275,
+      "tokens_seen": 2689138688
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.738565238324507e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3775765324641793,
+      "tokens_seen": 2689269760
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.737762798908683e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.377563701859556,
+      "tokens_seen": 2689400832
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.736960359492859e-05,
+      "loss": 1.2187,
+      "theoretical_loss": 3.377550872055316,
+      "tokens_seen": 2689531904
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.736157920077034e-05,
+      "loss": 1.2557,
+      "theoretical_loss": 3.37753804305137,
+      "tokens_seen": 2689662976
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.73535548066121e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.377525214847629,
+      "tokens_seen": 2689794048
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.734553041245386e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3775123874440047,
+      "tokens_seen": 2689925120
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.733750601829562e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.377499560840408,
+      "tokens_seen": 2690056192
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.732948162413738e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.37748673503675,
+      "tokens_seen": 2690187264
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.732145722997914e-05,
+      "loss": 1.2838,
+      "theoretical_loss": 3.3774739100329416,
+      "tokens_seen": 2690318336
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.73134328358209e-05,
+      "loss": 1.148,
+      "theoretical_loss": 3.377461085828894,
+      "tokens_seen": 2690449408
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7305408441662656e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.377448262424519,
+      "tokens_seen": 2690580480
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7297384047504416e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.3774354398197266,
+      "tokens_seen": 2690711552
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7289359653346176e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.377422618014429,
+      "tokens_seen": 2690842624
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7281335259187936e-05,
+      "loss": 1.155,
+      "theoretical_loss": 3.377409797008537,
+      "tokens_seen": 2690973696
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7273310865029695e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.377396976801962,
+      "tokens_seen": 2691104768
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7265286470871455e-05,
+      "loss": 1.1715,
+      "theoretical_loss": 3.3773841573946153,
+      "tokens_seen": 2691235840
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7257262076713215e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.377371338786408,
+      "tokens_seen": 2691366912
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.724923768255497e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.377358520977251,
+      "tokens_seen": 2691497984
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.724121328839673e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.3773457039670562,
+      "tokens_seen": 2691629056
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.723318889423849e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.3773328877557347,
+      "tokens_seen": 2691760128
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.722516450008025e-05,
+      "loss": 1.1741,
+      "theoretical_loss": 3.377320072343198,
+      "tokens_seen": 2691891200
+    },
+    {
+      "epoch": 0.63,
+      "objective/train/docs_used": 903428,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.222994089126587,
+      "objective/train/theoretical_loss": 3.377307257729357,
+      "objective/train/tokens_used": 1062481376,
+      "theoretical_loss": 3.377307257729357,
+      "tokens_seen": 2692022272
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.721714010592201e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.377307257729357,
+      "tokens_seen": 2692022272
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.720911571176377e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3772944439141233,
+      "tokens_seen": 2692153344
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.720109131760552e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.377281630897408,
+      "tokens_seen": 2692284416
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.719306692344728e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.3772688186791227,
+      "tokens_seen": 2692415488
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.718504252928904e-05,
+      "loss": 1.1897,
+      "theoretical_loss": 3.3772560072591786,
+      "tokens_seen": 2692546560
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.71770181351308e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.3772431966374876,
+      "tokens_seen": 2692677632
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.716899374097256e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.3772303868139604,
+      "tokens_seen": 2692808704
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.716096934681432e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.377217577788509,
+      "tokens_seen": 2692939776
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7152944952656074e-05,
+      "loss": 1.1539,
+      "theoretical_loss": 3.377204769561044,
+      "tokens_seen": 2693070848
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7144920558497834e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.3771919621314783,
+      "tokens_seen": 2693201920
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7136896164339593e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3771791554997224,
+      "tokens_seen": 2693332992
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7128871770181353e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.3771663496656874,
+      "tokens_seen": 2693464064
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.712084737602311e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.3771535446292855,
+      "tokens_seen": 2693595136
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.711282298186487e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.377140740390428,
+      "tokens_seen": 2693726208
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7104798587706626e-05,
+      "loss": 1.1695,
+      "theoretical_loss": 3.3771279369490266,
+      "tokens_seen": 2693857280
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7096774193548386e-05,
+      "loss": 1.3145,
+      "theoretical_loss": 3.3771151343049923,
+      "tokens_seen": 2693988352
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7088749799390146e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.377102332458237,
+      "tokens_seen": 2694119424
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7080725405231906e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.377089531408673,
+      "tokens_seen": 2694250496
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.7072701011073666e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.3770767311562104,
+      "tokens_seen": 2694381568
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.706467661691542e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.3770639317007616,
+      "tokens_seen": 2694512640
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.705665222275718e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.3770511330422384,
+      "tokens_seen": 2694643712
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.704862782859894e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3770383351805524,
+      "tokens_seen": 2694774784
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.70406034344407e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.3770255381156145,
+      "tokens_seen": 2694905856
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.703257904028246e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.377012741847337,
+      "tokens_seen": 2695036928
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.702455464612422e-05,
+      "loss": 1.2858,
+      "theoretical_loss": 3.3769999463756317,
+      "tokens_seen": 2695168000
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.701653025196598e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.3769871517004098,
+      "tokens_seen": 2695299072
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.700850585780773e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.376974357821583,
+      "tokens_seen": 2695430144
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.700048146364949e-05,
+      "loss": 1.3107,
+      "theoretical_loss": 3.3769615647390636,
+      "tokens_seen": 2695561216
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.699245706949125e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.3769487724527627,
+      "tokens_seen": 2695692288
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.698443267533301e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.3769359809625925,
+      "tokens_seen": 2695823360
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.697640828117477e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3769231902684638,
+      "tokens_seen": 2695954432
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.696838388701653e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.37691040037029,
+      "tokens_seen": 2696085504
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.696035949285829e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.376897611267981,
+      "tokens_seen": 2696216576
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6952335098700044e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.37688482296145,
+      "tokens_seen": 2696347648
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6944310704541804e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3768720354506083,
+      "tokens_seen": 2696478720
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6936286310383564e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.376859248735368,
+      "tokens_seen": 2696609792
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6928261916225324e-05,
+      "loss": 1.1997,
+      "theoretical_loss": 3.3768464628156405,
+      "tokens_seen": 2696740864
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6920237522067084e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3768336776913377,
+      "tokens_seen": 2696871936
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6912213127908844e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.376820893362372,
+      "tokens_seen": 2697003008
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6904188733750604e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.376808109828654,
+      "tokens_seen": 2697134080
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6896164339592364e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.3767953270900977,
+      "tokens_seen": 2697265152
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.688813994543412e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.3767825451466127,
+      "tokens_seen": 2697396224
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6880115551275877e-05,
+      "loss": 1.2983,
+      "theoretical_loss": 3.3767697639981122,
+      "tokens_seen": 2697527296
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 3.6872091157117637e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.3767569836445084,
+      "tokens_seen": 2697658368
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6864066762959396e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.3767442040857123,
+      "tokens_seen": 2697789440
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6856042368801156e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.3767314253216365,
+      "tokens_seen": 2697920512
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6848017974642916e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.3767186473521926,
+      "tokens_seen": 2698051584
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6839993580484676e-05,
+      "loss": 1.2327,
+      "theoretical_loss": 3.3767058701772927,
+      "tokens_seen": 2698182656
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.683196918632643e-05,
+      "loss": 1.1784,
+      "theoretical_loss": 3.376693093796849,
+      "tokens_seen": 2698313728
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.682394479216819e-05,
+      "loss": 1.1866,
+      "theoretical_loss": 3.3766803182107736,
+      "tokens_seen": 2698444800
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 905717,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2821831703186035,
+      "objective/train/theoretical_loss": 3.376667543418978,
+      "objective/train/tokens_used": 1069034976,
+      "theoretical_loss": 3.376667543418978,
+      "tokens_seen": 2698575872
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.681592039800995e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.376667543418978,
+      "tokens_seen": 2698575872
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.680789600385171e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.376654769421375,
+      "tokens_seen": 2698706944
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.679987160969347e-05,
+      "loss": 1.2856,
+      "theoretical_loss": 3.3766419962178755,
+      "tokens_seen": 2698838016
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.679184721553523e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.376629223808393,
+      "tokens_seen": 2698969088
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.678382282137699e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.376616452192838,
+      "tokens_seen": 2699100160
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.677579842721875e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.376603681371124,
+      "tokens_seen": 2699231232
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.67677740330605e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.376590911343163,
+      "tokens_seen": 2699362304
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.675974963890226e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3765781421088663,
+      "tokens_seen": 2699493376
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.675172524474402e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.3765653736681465,
+      "tokens_seen": 2699624448
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.674370085058578e-05,
+      "loss": 1.2766,
+      "theoretical_loss": 3.3765526060209154,
+      "tokens_seen": 2699755520
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.673567645642754e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.376539839167086,
+      "tokens_seen": 2699886592
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.67276520622693e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.37652707310657,
+      "tokens_seen": 2700017664
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.671962766811106e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.376514307839279,
+      "tokens_seen": 2700148736
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6711603273952814e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.376501543365126,
+      "tokens_seen": 2700279808
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6703578879794574e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.376488779684023,
+      "tokens_seen": 2700410880
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6695554485636334e-05,
+      "loss": 1.3152,
+      "theoretical_loss": 3.3764760167958823,
+      "tokens_seen": 2700541952
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6687530091478094e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.376463254700616,
+      "tokens_seen": 2700673024
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6679505697319854e-05,
+      "loss": 1.2327,
+      "theoretical_loss": 3.3764504933981363,
+      "tokens_seen": 2700804096
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6671481303161614e-05,
+      "loss": 1.2968,
+      "theoretical_loss": 3.376437732888356,
+      "tokens_seen": 2700935168
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6663456909003374e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.376424973171187,
+      "tokens_seen": 2701066240
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.665543251484513e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.376412214246541,
+      "tokens_seen": 2701197312
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.664740812068689e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.376399456114332,
+      "tokens_seen": 2701328384
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.663938372652865e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.3763866987744704,
+      "tokens_seen": 2701459456
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6631359332370407e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.37637394222687,
+      "tokens_seen": 2701590528
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6623334938212166e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.376361186471442,
+      "tokens_seen": 2701721600
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6615310544053926e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.3763484315080996,
+      "tokens_seen": 2701852672
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6607286149895686e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.376335677336755,
+      "tokens_seen": 2701983744
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6599261755737446e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.376322923957321,
+      "tokens_seen": 2702114816
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.65912373615792e-05,
+      "loss": 1.32,
+      "theoretical_loss": 3.3763101713697092,
+      "tokens_seen": 2702245888
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.658321296742096e-05,
+      "loss": 1.2438,
+      "theoretical_loss": 3.3762974195738327,
+      "tokens_seen": 2702376960
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.657518857326272e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.3762846685696033,
+      "tokens_seen": 2702508032
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.656716417910448e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.3762719183569345,
+      "tokens_seen": 2702639104
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.655913978494624e-05,
+      "loss": 1.3038,
+      "theoretical_loss": 3.3762591689357375,
+      "tokens_seen": 2702770176
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6551115390788e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.3762464203059257,
+      "tokens_seen": 2702901248
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.654309099662976e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.376233672467411,
+      "tokens_seen": 2703032320
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.653506660247151e-05,
+      "loss": 1.1668,
+      "theoretical_loss": 3.3762209254201068,
+      "tokens_seen": 2703163392
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.652704220831327e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.3762081791639247,
+      "tokens_seen": 2703294464
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.651901781415503e-05,
+      "loss": 1.3145,
+      "theoretical_loss": 3.3761954336987774,
+      "tokens_seen": 2703425536
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.651099341999679e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.3761826890245783,
+      "tokens_seen": 2703556608
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.650296902583855e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.376169945141239,
+      "tokens_seen": 2703687680
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.649494463168031e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.376157202048672,
+      "tokens_seen": 2703818752
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.648692023752207e-05,
+      "loss": 1.2481,
+      "theoretical_loss": 3.376144459746791,
+      "tokens_seen": 2703949824
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6478895843363824e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.3761317182355075,
+      "tokens_seen": 2704080896
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6470871449205584e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.376118977514735,
+      "tokens_seen": 2704211968
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6462847055047344e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3761062375843855,
+      "tokens_seen": 2704343040
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6454822660889104e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.376093498444372,
+      "tokens_seen": 2704474112
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6446798266730864e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.376080760094607,
+      "tokens_seen": 2704605184
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6438773872572624e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.3760680225350033,
+      "tokens_seen": 2704736256
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6430749478414384e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.376055285765473,
+      "tokens_seen": 2704867328
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6422725084256144e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3760425497859297,
+      "tokens_seen": 2704998400
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 908174,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.329195261001587,
+      "objective/train/theoretical_loss": 3.376029814596286,
+      "objective/train/tokens_used": 1075588576,
+      "theoretical_loss": 3.376029814596286,
+      "tokens_seen": 2705129472
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.64147006900979e-05,
+      "loss": 1.2991,
+      "theoretical_loss": 3.376029814596286,
+      "tokens_seen": 2705129472
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.640667629593966e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.3760170801964544,
+      "tokens_seen": 2705260544
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.639865190178142e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.376004346586347,
+      "tokens_seen": 2705391616
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.639062750762318e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.375991613765878,
+      "tokens_seen": 2705522688
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6382603113464937e-05,
+      "loss": 1.2902,
+      "theoretical_loss": 3.375978881734959,
+      "tokens_seen": 2705653760
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6374578719306696e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.375966150493504,
+      "tokens_seen": 2705784832
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6366554325148456e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.375953420041424,
+      "tokens_seen": 2705915904
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.635852993099021e-05,
+      "loss": 1.1535,
+      "theoretical_loss": 3.3759406903786333,
+      "tokens_seen": 2706046976
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.635050553683197e-05,
+      "loss": 1.2188,
+      "theoretical_loss": 3.3759279615050444,
+      "tokens_seen": 2706178048
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.634248114267373e-05,
+      "loss": 1.1512,
+      "theoretical_loss": 3.3759152334205695,
+      "tokens_seen": 2706309120
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.633445674851549e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.375902506125122,
+      "tokens_seen": 2706440192
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.632643235435725e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.3758897796186154,
+      "tokens_seen": 2706571264
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.631840796019901e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.375877053900962,
+      "tokens_seen": 2706702336
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.631038356604077e-05,
+      "loss": 1.3138,
+      "theoretical_loss": 3.3758643289720744,
+      "tokens_seen": 2706833408
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.630235917188253e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.3758516048318654,
+      "tokens_seen": 2706964480
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.629433477772428e-05,
+      "loss": 1.1591,
+      "theoretical_loss": 3.3758388814802487,
+      "tokens_seen": 2707095552
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.628631038356604e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3758261589171368,
+      "tokens_seen": 2707226624
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.62782859894078e-05,
+      "loss": 1.182,
+      "theoretical_loss": 3.375813437142443,
+      "tokens_seen": 2707357696
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.627026159524956e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.3758007161560797,
+      "tokens_seen": 2707488768
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.626223720109132e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.3757879959579604,
+      "tokens_seen": 2707619840
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.625421280693308e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.375775276547998,
+      "tokens_seen": 2707750912
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.624618841277484e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.375762557926105,
+      "tokens_seen": 2707881984
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6238164018616595e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.375749840092195,
+      "tokens_seen": 2708013056
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6230139624458354e-05,
+      "loss": 1.2846,
+      "theoretical_loss": 3.3757371230461812,
+      "tokens_seen": 2708144128
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6222115230300114e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.3757244067879757,
+      "tokens_seen": 2708275200
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6214090836141874e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.375711691317493,
+      "tokens_seen": 2708406272
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6206066441983634e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.375698976634645,
+      "tokens_seen": 2708537344
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6198042047825394e-05,
+      "loss": 1.3113,
+      "theoretical_loss": 3.375686262739345,
+      "tokens_seen": 2708668416
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6190017653667154e-05,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3756735496315065,
+      "tokens_seen": 2708799488
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.618199325950891e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3756608373110426,
+      "tokens_seen": 2708930560
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.617396886535067e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.3756481257778663,
+      "tokens_seen": 2709061632
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.616594447119243e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.3756354150318906,
+      "tokens_seen": 2709192704
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.615792007703419e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.375622705073029,
+      "tokens_seen": 2709323776
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.614989568287595e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3756099959011943,
+      "tokens_seen": 2709454848
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.614187128871771e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.3755972875163,
+      "tokens_seen": 2709585920
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6133846894559467e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3755845799182596,
+      "tokens_seen": 2709716992
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6125822500401226e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3755718731069853,
+      "tokens_seen": 2709848064
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.611779810624298e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.375559167082391,
+      "tokens_seen": 2709979136
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.610977371208474e-05,
+      "loss": 1.2833,
+      "theoretical_loss": 3.3755464618443902,
+      "tokens_seen": 2710110208
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.61017493179265e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.375533757392896,
+      "tokens_seen": 2710241280
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.609372492376826e-05,
+      "loss": 1.159,
+      "theoretical_loss": 3.3755210537278213,
+      "tokens_seen": 2710372352
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.608570052961002e-05,
+      "loss": 1.1826,
+      "theoretical_loss": 3.3755083508490795,
+      "tokens_seen": 2710503424
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.607767613545178e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3754956487565844,
+      "tokens_seen": 2710634496
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.606965174129354e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.375482947450249,
+      "tokens_seen": 2710765568
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.606162734713529e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3754702469299867,
+      "tokens_seen": 2710896640
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.605360295297705e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.37545754719571,
+      "tokens_seen": 2711027712
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.604557855881881e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3754448482473336,
+      "tokens_seen": 2711158784
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.603755416466057e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.3754321500847704,
+      "tokens_seen": 2711289856
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.602952977050233e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.3754194527079333,
+      "tokens_seen": 2711420928
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.602150537634409e-05,
+      "loss": 1.2768,
+      "theoretical_loss": 3.3754067561167367,
+      "tokens_seen": 2711552000
+    },
+    {
+      "epoch": 0.64,
+      "objective/train/docs_used": 910231,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2605091333389282,
+      "objective/train/theoretical_loss": 3.375394060311093,
+      "objective/train/tokens_used": 1082142176,
+      "theoretical_loss": 3.375394060311093,
+      "tokens_seen": 2711683072
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.601348098218585e-05,
+      "loss": 1.2955,
+      "theoretical_loss": 3.375394060311093,
+      "tokens_seen": 2711683072
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.6005456588027605e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3753813652909157,
+      "tokens_seen": 2711814144
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5997432193869365e-05,
+      "loss": 1.1839,
+      "theoretical_loss": 3.375368671056119,
+      "tokens_seen": 2711945216
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5989407799711125e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.375355977606616,
+      "tokens_seen": 2712076288
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5981383405552884e-05,
+      "loss": 1.289,
+      "theoretical_loss": 3.37534328494232,
+      "tokens_seen": 2712207360
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5973359011394644e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.3753305930631443,
+      "tokens_seen": 2712338432
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5965334617236404e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3753179019690034,
+      "tokens_seen": 2712469504
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5957310223078164e-05,
+      "loss": 1.2986,
+      "theoretical_loss": 3.3753052116598097,
+      "tokens_seen": 2712600576
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.5949285828919924e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.375292522135477,
+      "tokens_seen": 2712731648
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.594126143476168e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.375279833395919,
+      "tokens_seen": 2712862720
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.593323704060344e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.3752671454410494,
+      "tokens_seen": 2712993792
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.59252126464452e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3752544582707813,
+      "tokens_seen": 2713124864
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.591718825228696e-05,
+      "loss": 1.1815,
+      "theoretical_loss": 3.3752417718850287,
+      "tokens_seen": 2713255936
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.590916385812872e-05,
+      "loss": 1.3173,
+      "theoretical_loss": 3.375229086283705,
+      "tokens_seen": 2713387008
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.590113946397048e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.375216401466724,
+      "tokens_seen": 2713518080
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.589311506981223e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.3752037174339993,
+      "tokens_seen": 2713649152
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.588509067565399e-05,
+      "loss": 1.2664,
+      "theoretical_loss": 3.3751910341854443,
+      "tokens_seen": 2713780224
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.587706628149575e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3751783517209724,
+      "tokens_seen": 2713911296
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.586904188733751e-05,
+      "loss": 1.1808,
+      "theoretical_loss": 3.375165670040498,
+      "tokens_seen": 2714042368
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.586101749317927e-05,
+      "loss": 1.1916,
+      "theoretical_loss": 3.3751529891439347,
+      "tokens_seen": 2714173440
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.585299309902103e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3751403090311958,
+      "tokens_seen": 2714304512
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.584496870486278e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.375127629702195,
+      "tokens_seen": 2714435584
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.583694431070454e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3751149511568457,
+      "tokens_seen": 2714566656
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.58289199165463e-05,
+      "loss": 1.133,
+      "theoretical_loss": 3.3751022733950626,
+      "tokens_seen": 2714697728
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.582089552238806e-05,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3750895964167587,
+      "tokens_seen": 2714828800
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.581287112822982e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.375076920221848,
+      "tokens_seen": 2714959872
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.580484673407158e-05,
+      "loss": 1.1462,
+      "theoretical_loss": 3.375064244810244,
+      "tokens_seen": 2715090944
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5796822339913335e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.375051570181861,
+      "tokens_seen": 2715222016
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5788797945755095e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3750388963366125,
+      "tokens_seen": 2715353088
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5780773551596855e-05,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3750262232744124,
+      "tokens_seen": 2715484160
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5772749157438615e-05,
+      "loss": 1.1487,
+      "theoretical_loss": 3.3750135509951744,
+      "tokens_seen": 2715615232
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5764724763280375e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.375000879498812,
+      "tokens_seen": 2715746304
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5756700369122135e-05,
+      "loss": 1.1724,
+      "theoretical_loss": 3.37498820878524,
+      "tokens_seen": 2715877376
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.574867597496389e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.3749755388543714,
+      "tokens_seen": 2716008448
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.574065158080565e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3749628697061205,
+      "tokens_seen": 2716139520
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.573262718664741e-05,
+      "loss": 1.1691,
+      "theoretical_loss": 3.3749502013404014,
+      "tokens_seen": 2716270592
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.572460279248917e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3749375337571275,
+      "tokens_seen": 2716401664
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.571657839833093e-05,
+      "loss": 1.2049,
+      "theoretical_loss": 3.3749248669562126,
+      "tokens_seen": 2716532736
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.570855400417269e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.374912200937571,
+      "tokens_seen": 2716663808
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.570052961001444e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.3748995357011173,
+      "tokens_seen": 2716794880
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.56925052158562e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.374886871246764,
+      "tokens_seen": 2716925952
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.568448082169796e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3748742075744262,
+      "tokens_seen": 2717057024
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.567645642753972e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3748615446840176,
+      "tokens_seen": 2717188096
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.566843203338148e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.374848882575452,
+      "tokens_seen": 2717319168
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.566040763922324e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3748362212486436,
+      "tokens_seen": 2717450240
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5652383245065e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3748235607035064,
+      "tokens_seen": 2717581312
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.564435885090675e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.3748109009399543,
+      "tokens_seen": 2717712384
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.563633445674851e-05,
+      "loss": 1.1494,
+      "theoretical_loss": 3.374798241957902,
+      "tokens_seen": 2717843456
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.562831006259027e-05,
+      "loss": 1.1729,
+      "theoretical_loss": 3.3747855837572622,
+      "tokens_seen": 2717974528
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.562028566843203e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3747729263379505,
+      "tokens_seen": 2718105600
+    },
+    {
+      "epoch": 0.65,
+      "objective/train/docs_used": 912432,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1833117008209229,
+      "objective/train/theoretical_loss": 3.3747602696998804,
+      "objective/train/tokens_used": 1088695776,
+      "theoretical_loss": 3.3747602696998804,
+      "tokens_seen": 2718236672
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.561226127427379e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3747602696998804,
+      "tokens_seen": 2718236672
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.560423688011555e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3747476138429655,
+      "tokens_seen": 2718367744
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.559621248595731e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.374734958767121,
+      "tokens_seen": 2718498816
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5588188091799066e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.37472230447226,
+      "tokens_seen": 2718629888
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5580163697640826e-05,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3747096509582972,
+      "tokens_seen": 2718760960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5572139303482585e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.3746969982251467,
+      "tokens_seen": 2718892032
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5564114909324345e-05,
+      "loss": 1.1453,
+      "theoretical_loss": 3.3746843462727227,
+      "tokens_seen": 2719023104
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5556090515166105e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.374671695100939,
+      "tokens_seen": 2719154176
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5548066121007865e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.3746590447097105,
+      "tokens_seen": 2719285248
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5540041726849625e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3746463950989507,
+      "tokens_seen": 2719416320
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.553201733269138e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3746337462685743,
+      "tokens_seen": 2719547392
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.552399293853314e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.374621098218496,
+      "tokens_seen": 2719678464
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.55159685443749e-05,
+      "loss": 1.1536,
+      "theoretical_loss": 3.374608450948629,
+      "tokens_seen": 2719809536
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.550794415021666e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3745958044588877,
+      "tokens_seen": 2719940608
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.549991975605842e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3745831587491875,
+      "tokens_seen": 2720071680
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.549189536190018e-05,
+      "loss": 1.2338,
+      "theoretical_loss": 3.3745705138194415,
+      "tokens_seen": 2720202752
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.548387096774194e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.3745578696695646,
+      "tokens_seen": 2720333824
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.54758465735837e-05,
+      "loss": 1.1547,
+      "theoretical_loss": 3.374545226299471,
+      "tokens_seen": 2720464896
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.546782217942545e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.374532583709075,
+      "tokens_seen": 2720595968
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.545979778526721e-05,
+      "loss": 1.1153,
+      "theoretical_loss": 3.374519941898291,
+      "tokens_seen": 2720727040
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.545177339110897e-05,
+      "loss": 1.1835,
+      "theoretical_loss": 3.374507300867034,
+      "tokens_seen": 2720858112
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.544374899695073e-05,
+      "loss": 1.172,
+      "theoretical_loss": 3.374494660615217,
+      "tokens_seen": 2720989184
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.543572460279249e-05,
+      "loss": 1.1709,
+      "theoretical_loss": 3.3744820211427555,
+      "tokens_seen": 2721120256
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.542770020863425e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.3744693824495635,
+      "tokens_seen": 2721251328
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.541967581447601e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.3744567445355553,
+      "tokens_seen": 2721382400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.541165142031776e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.3744441074006457,
+      "tokens_seen": 2721513472
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.540362702615952e-05,
+      "loss": 1.3505,
+      "theoretical_loss": 3.3744314710447494,
+      "tokens_seen": 2721644544
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.539560263200128e-05,
+      "loss": 1.1597,
+      "theoretical_loss": 3.3744188354677798,
+      "tokens_seen": 2721775616
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.538757823784304e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3744062006696525,
+      "tokens_seen": 2721906688
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.53795538436848e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3743935666502813,
+      "tokens_seen": 2722037760
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.537152944952656e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.374380933409581,
+      "tokens_seen": 2722168832
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.536350505536832e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.374368300947466,
+      "tokens_seen": 2722299904
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.535548066121008e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.374355669263851,
+      "tokens_seen": 2722430976
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5347456267051836e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.37434303835865,
+      "tokens_seen": 2722562048
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5339431872893596e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.3743304082317787,
+      "tokens_seen": 2722693120
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5331407478735356e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.3743177788831504,
+      "tokens_seen": 2722824192
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5323383084577115e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.37430515031268,
+      "tokens_seen": 2722955264
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5315358690418875e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.374292522520283,
+      "tokens_seen": 2723086336
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5307334296260635e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3742798955058735,
+      "tokens_seen": 2723217408
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5299309902102395e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.3742672692693656,
+      "tokens_seen": 2723348480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.529128550794415e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3742546438106737,
+      "tokens_seen": 2723479552
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.528326111378591e-05,
+      "loss": 1.2165,
+      "theoretical_loss": 3.374242019129714,
+      "tokens_seen": 2723610624
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.527523671962767e-05,
+      "loss": 1.2962,
+      "theoretical_loss": 3.3742293952263998,
+      "tokens_seen": 2723741696
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.526721232546943e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.3742167721006466,
+      "tokens_seen": 2723872768
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.525918793131119e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3742041497523685,
+      "tokens_seen": 2724003840
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.525116353715295e-05,
+      "loss": 1.1531,
+      "theoretical_loss": 3.3741915281814805,
+      "tokens_seen": 2724134912
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.524313914299471e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.374178907387897,
+      "tokens_seen": 2724265984
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.523511474883646e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.3741662873715335,
+      "tokens_seen": 2724397056
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.522709035467822e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.374153668132304,
+      "tokens_seen": 2724528128
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.521906596051998e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.374141049670123,
+      "tokens_seen": 2724659200
+    },
+    {
+      "epoch": 0.65,
+      "objective/train/docs_used": 914654,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2970740795135498,
+      "objective/train/theoretical_loss": 3.3741284319849063,
+      "objective/train/tokens_used": 1095249376,
+      "theoretical_loss": 3.3741284319849063,
+      "tokens_seen": 2724790272
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.521104156636174e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.3741284319849063,
+      "tokens_seen": 2724790272
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.52030171722035e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3741158150765678,
+      "tokens_seen": 2724921344
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.519499277804526e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.374103198945023,
+      "tokens_seen": 2725052416
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.518696838388702e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.374090583590186,
+      "tokens_seen": 2725183488
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.517894398972878e-05,
+      "loss": 1.3146,
+      "theoretical_loss": 3.3740779690119727,
+      "tokens_seen": 2725314560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.517091959557053e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3740653552102966,
+      "tokens_seen": 2725445632
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.516289520141229e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.3740527421850732,
+      "tokens_seen": 2725576704
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.515487080725405e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3740401299362177,
+      "tokens_seen": 2725707776
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.514684641309581e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.374027518463645,
+      "tokens_seen": 2725838848
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.513882201893757e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3740149077672688,
+      "tokens_seen": 2725969920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.513079762477933e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3740022978470052,
+      "tokens_seen": 2726100992
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.512277323062109e-05,
+      "loss": 1.1668,
+      "theoretical_loss": 3.373989688702769,
+      "tokens_seen": 2726232064
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5114748836462846e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.3739770803344746,
+      "tokens_seen": 2726363136
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5106724442304606e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3739644727420375,
+      "tokens_seen": 2726494208
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5098700048146366e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.373951865925372,
+      "tokens_seen": 2726625280
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5090675653988126e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.373939259884394,
+      "tokens_seen": 2726756352
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5082651259829885e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3739266546190176,
+      "tokens_seen": 2726887424
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5074626865671645e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.3739140501291587,
+      "tokens_seen": 2727018496
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.5066602471513405e-05,
+      "loss": 1.1955,
+      "theoretical_loss": 3.3739014464147314,
+      "tokens_seen": 2727149568
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.505857807735516e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.373888843475651,
+      "tokens_seen": 2727280640
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.505055368319692e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.3738762413118333,
+      "tokens_seen": 2727411712
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.504252928903868e-05,
+      "loss": 1.2088,
+      "theoretical_loss": 3.3738636399231923,
+      "tokens_seen": 2727542784
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.503450489488044e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3738510393096437,
+      "tokens_seen": 2727673856
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.50264805007222e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3738384394711023,
+      "tokens_seen": 2727804928
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.501845610656396e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.3738258404074832,
+      "tokens_seen": 2727936000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.501043171240572e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3738132421187017,
+      "tokens_seen": 2728067072
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.500240731824748e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.373800644604673,
+      "tokens_seen": 2728198144
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.499438292408923e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3737880478653115,
+      "tokens_seen": 2728329216
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.498635852993099e-05,
+      "loss": 1.3318,
+      "theoretical_loss": 3.373775451900533,
+      "tokens_seen": 2728460288
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.497833413577275e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.373762856710253,
+      "tokens_seen": 2728591360
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.497030974161451e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.3737502622943865,
+      "tokens_seen": 2728722432
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.496228534745627e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.373737668652848,
+      "tokens_seen": 2728853504
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.495426095329803e-05,
+      "loss": 1.2799,
+      "theoretical_loss": 3.3737250757855533,
+      "tokens_seen": 2728984576
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.494623655913979e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.373712483692417,
+      "tokens_seen": 2729115648
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4938212164981543e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3736998923733554,
+      "tokens_seen": 2729246720
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.49301877708233e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.373687301828283,
+      "tokens_seen": 2729377792
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.492216337666506e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.3736747120571153,
+      "tokens_seen": 2729508864
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.491413898250682e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.373662123059767,
+      "tokens_seen": 2729639936
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.490611458834858e-05,
+      "loss": 1.2676,
+      "theoretical_loss": 3.3736495348361544,
+      "tokens_seen": 2729771008
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.489809019419034e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.373636947386192,
+      "tokens_seen": 2729902080
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.48900658000321e-05,
+      "loss": 1.2851,
+      "theoretical_loss": 3.3736243607097953,
+      "tokens_seen": 2730033152
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.488204140587386e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.3736117748068795,
+      "tokens_seen": 2730164224
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4874017011715616e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3735991896773605,
+      "tokens_seen": 2730295296
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4865992617557376e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.373586605321153,
+      "tokens_seen": 2730426368
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4857968223399136e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.373574021738173,
+      "tokens_seen": 2730557440
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 3.4849943829240896e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.373561438928335,
+      "tokens_seen": 2730688512
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4841919435082656e-05,
+      "loss": 1.1961,
+      "theoretical_loss": 3.3735488568915555,
+      "tokens_seen": 2730819584
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4833895040924415e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.373536275627749,
+      "tokens_seen": 2730950656
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4825870646766175e-05,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3735236951368313,
+      "tokens_seen": 2731081728
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.481784625260793e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.373511115418718,
+      "tokens_seen": 2731212800
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 916965,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1878814697265625,
+      "objective/train/theoretical_loss": 3.3734985364733237,
+      "objective/train/tokens_used": 1101802976,
+      "theoretical_loss": 3.3734985364733237,
+      "tokens_seen": 2731343872
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.480982185844969e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3734985364733237,
+      "tokens_seen": 2731343872
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.480179746429145e-05,
+      "loss": 1.1397,
+      "theoretical_loss": 3.3734859583005647,
+      "tokens_seen": 2731474944
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.479377307013321e-05,
+      "loss": 1.159,
+      "theoretical_loss": 3.3734733809003563,
+      "tokens_seen": 2731606016
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.478574867597497e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.3734608042726135,
+      "tokens_seen": 2731737088
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.477772428181673e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.373448228417253,
+      "tokens_seen": 2731868160
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.476969988765849e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.373435653334189,
+      "tokens_seen": 2731999232
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.476167549350024e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.3734230790233375,
+      "tokens_seen": 2732130304
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4753651099342e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3734105054846144,
+      "tokens_seen": 2732261376
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.474562670518376e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.3733979327179346,
+      "tokens_seen": 2732392448
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.473760231102552e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.3733853607232143,
+      "tokens_seen": 2732523520
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.472957791686728e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.3733727895003685,
+      "tokens_seen": 2732654592
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.472155352270904e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.3733602190493133,
+      "tokens_seen": 2732785664
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.47135291285508e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.373347649369964,
+      "tokens_seen": 2732916736
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.470550473439256e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.373335080462236,
+      "tokens_seen": 2733047808
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4697480340234314e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.3733225123260455,
+      "tokens_seen": 2733178880
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4689455946076073e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3733099449613078,
+      "tokens_seen": 2733309952
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.468143155191783e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.3732973783679387,
+      "tokens_seen": 2733441024
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.467340715775959e-05,
+      "loss": 1.1775,
+      "theoretical_loss": 3.3732848125458537,
+      "tokens_seen": 2733572096
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.466538276360135e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.3732722474949686,
+      "tokens_seen": 2733703168
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.465735836944311e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3732596832151986,
+      "tokens_seen": 2733834240
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.464933397528487e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.3732471197064604,
+      "tokens_seen": 2733965312
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4641309581126626e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.373234556968669,
+      "tokens_seen": 2734096384
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4633285186968386e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.3732219950017406,
+      "tokens_seen": 2734227456
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4625260792810146e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3732094338055902,
+      "tokens_seen": 2734358528
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4617236398651906e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.373196873380134,
+      "tokens_seen": 2734489600
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4609212004493666e-05,
+      "loss": 1.3137,
+      "theoretical_loss": 3.373184313725288,
+      "tokens_seen": 2734620672
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4601187610335426e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.373171754840968,
+      "tokens_seen": 2734751744
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4593163216177186e-05,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3731591967270895,
+      "tokens_seen": 2734882816
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.458513882201894e-05,
+      "loss": 1.2961,
+      "theoretical_loss": 3.3731466393835685,
+      "tokens_seen": 2735013888
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.45771144278607e-05,
+      "loss": 1.1718,
+      "theoretical_loss": 3.3731340828103207,
+      "tokens_seen": 2735144960
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.456909003370246e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.373121527007262,
+      "tokens_seen": 2735276032
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.456106563954422e-05,
+      "loss": 1.1853,
+      "theoretical_loss": 3.3731089719743084,
+      "tokens_seen": 2735407104
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.455304124538598e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.373096417711375,
+      "tokens_seen": 2735538176
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.454501685122774e-05,
+      "loss": 1.3246,
+      "theoretical_loss": 3.373083864218379,
+      "tokens_seen": 2735669248
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.453699245706949e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.3730713114952353,
+      "tokens_seen": 2735800320
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.452896806291125e-05,
+      "loss": 1.175,
+      "theoretical_loss": 3.37305875954186,
+      "tokens_seen": 2735931392
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.452094366875301e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3730462083581694,
+      "tokens_seen": 2736062464
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.451291927459477e-05,
+      "loss": 1.2849,
+      "theoretical_loss": 3.3730336579440787,
+      "tokens_seen": 2736193536
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.450489488043653e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3730211082995045,
+      "tokens_seen": 2736324608
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.449687048627829e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.373008559424363,
+      "tokens_seen": 2736455680
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4488846092120044e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.372996011318569,
+      "tokens_seen": 2736586752
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4480821697961804e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.3729834639820395,
+      "tokens_seen": 2736717824
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4472797303803564e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.3729709174146905,
+      "tokens_seen": 2736848896
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4464772909645324e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.3729583716164377,
+      "tokens_seen": 2736979968
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4456748515487084e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.372945826587197,
+      "tokens_seen": 2737111040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4448724121328844e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.3729332823268847,
+      "tokens_seen": 2737242112
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.44406997271706e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3729207388354165,
+      "tokens_seen": 2737373184
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4432675333012357e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.372908196112709,
+      "tokens_seen": 2737504256
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4424650938854116e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3728956541586776,
+      "tokens_seen": 2737635328
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4416626544695876e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.3728831129732395,
+      "tokens_seen": 2737766400
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 919044,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1700243949890137,
+      "objective/train/theoretical_loss": 3.3728705725563097,
+      "objective/train/tokens_used": 1108356576,
+      "theoretical_loss": 3.3728705725563097,
+      "tokens_seen": 2737897472
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4408602150537636e-05,
+      "loss": 1.1723,
+      "theoretical_loss": 3.3728705725563097,
+      "tokens_seen": 2737897472
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4400577756379396e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.3728580329078044,
+      "tokens_seen": 2738028544
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.439255336222115e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3728454940276404,
+      "tokens_seen": 2738159616
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.438452896806291e-05,
+      "loss": 1.3756,
+      "theoretical_loss": 3.3728329559157335,
+      "tokens_seen": 2738290688
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.437650457390467e-05,
+      "loss": 1.3022,
+      "theoretical_loss": 3.372820418572,
+      "tokens_seen": 2738421760
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.436848017974643e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.372807881996356,
+      "tokens_seen": 2738552832
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.436045578558819e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.372795346188717,
+      "tokens_seen": 2738683904
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.435243139142995e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.3727828111490004,
+      "tokens_seen": 2738814976
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.43444069972717e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3727702768771213,
+      "tokens_seen": 2738946048
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.433638260311346e-05,
+      "loss": 1.2318,
+      "theoretical_loss": 3.372757743372997,
+      "tokens_seen": 2739077120
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.432835820895522e-05,
+      "loss": 1.2221,
+      "theoretical_loss": 3.372745210636543,
+      "tokens_seen": 2739208192
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.432033381479698e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3727326786676755,
+      "tokens_seen": 2739339264
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.431230942063874e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.3727201474663113,
+      "tokens_seen": 2739470336
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.43042850264805e-05,
+      "loss": 1.2808,
+      "theoretical_loss": 3.372707617032366,
+      "tokens_seen": 2739601408
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.429626063232226e-05,
+      "loss": 1.2876,
+      "theoretical_loss": 3.3726950873657566,
+      "tokens_seen": 2739732480
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4288236238164015e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.372682558466399,
+      "tokens_seen": 2739863552
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4280211844005774e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.3726700303342096,
+      "tokens_seen": 2739994624
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4272187449847534e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.372657502969105,
+      "tokens_seen": 2740125696
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4264163055689294e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.3726449763710007,
+      "tokens_seen": 2740256768
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4256138661531054e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.3726324505398138,
+      "tokens_seen": 2740387840
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4248114267372814e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.372619925475461,
+      "tokens_seen": 2740518912
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4240089873214574e-05,
+      "loss": 1.1712,
+      "theoretical_loss": 3.3726074011778575,
+      "tokens_seen": 2740649984
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4232065479056334e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.372594877646921,
+      "tokens_seen": 2740781056
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.422404108489809e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3725823548825664,
+      "tokens_seen": 2740912128
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.421601669073985e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.372569832884712,
+      "tokens_seen": 2741043200
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.420799229658161e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.372557311653272,
+      "tokens_seen": 2741174272
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.419996790242337e-05,
+      "loss": 1.2077,
+      "theoretical_loss": 3.372544791188165,
+      "tokens_seen": 2741305344
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.419194350826513e-05,
+      "loss": 1.1939,
+      "theoretical_loss": 3.3725322714893062,
+      "tokens_seen": 2741436416
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4183919114106887e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.372519752556612,
+      "tokens_seen": 2741567488
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4175894719948646e-05,
+      "loss": 1.1965,
+      "theoretical_loss": 3.3725072343900004,
+      "tokens_seen": 2741698560
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.41678703257904e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.372494716989386,
+      "tokens_seen": 2741829632
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.415984593163216e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.3724822003546864,
+      "tokens_seen": 2741960704
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.415182153747392e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.3724696844858175,
+      "tokens_seen": 2742091776
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.414379714331568e-05,
+      "loss": 1.3142,
+      "theoretical_loss": 3.3724571693826966,
+      "tokens_seen": 2742222848
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.413577274915744e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3724446550452396,
+      "tokens_seen": 2742353920
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.41277483549992e-05,
+      "loss": 1.3164,
+      "theoretical_loss": 3.372432141473363,
+      "tokens_seen": 2742484992
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.411972396084096e-05,
+      "loss": 1.322,
+      "theoretical_loss": 3.372419628666984,
+      "tokens_seen": 2742616064
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.411169956668271e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.372407116626019,
+      "tokens_seen": 2742747136
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.410367517252447e-05,
+      "loss": 1.3088,
+      "theoretical_loss": 3.3723946053503844,
+      "tokens_seen": 2742878208
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.409565077836623e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.3723820948399967,
+      "tokens_seen": 2743009280
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.408762638420799e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.3723695850947726,
+      "tokens_seen": 2743140352
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.407960199004975e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.3723570761146293,
+      "tokens_seen": 2743271424
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.407157759589151e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.3723445678994826,
+      "tokens_seen": 2743402496
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.406355320173327e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3723320604492497,
+      "tokens_seen": 2743533568
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.405552880757503e-05,
+      "loss": 1.1879,
+      "theoretical_loss": 3.3723195537638473,
+      "tokens_seen": 2743664640
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4047504413416785e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.372307047843192,
+      "tokens_seen": 2743795712
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4039480019258545e-05,
+      "loss": 1.2734,
+      "theoretical_loss": 3.3722945426872,
+      "tokens_seen": 2743926784
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4031455625100304e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.372282038295789,
+      "tokens_seen": 2744057856
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4023431230942064e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.372269534668875,
+      "tokens_seen": 2744188928
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4015406836783824e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.372257031806375,
+      "tokens_seen": 2744320000
+    },
+    {
+      "epoch": 0.66,
+      "objective/train/docs_used": 921178,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2632722854614258,
+      "objective/train/theoretical_loss": 3.372244529708206,
+      "objective/train/tokens_used": 1114910176,
+      "theoretical_loss": 3.372244529708206,
+      "tokens_seen": 2744451072
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.4007382442625584e-05,
+      "loss": 1.2976,
+      "theoretical_loss": 3.372244529708206,
+      "tokens_seen": 2744451072
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.3999358048467344e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.372232028374284,
+      "tokens_seen": 2744582144
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.39913336543091e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3722195278045266,
+      "tokens_seen": 2744713216
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.398330926015086e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.3722070279988507,
+      "tokens_seen": 2744844288
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.397528486599262e-05,
+      "loss": 1.2838,
+      "theoretical_loss": 3.3721945289571718,
+      "tokens_seen": 2744975360
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.396726047183438e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.3721820306794084,
+      "tokens_seen": 2745106432
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.395923607767614e-05,
+      "loss": 1.1825,
+      "theoretical_loss": 3.372169533165476,
+      "tokens_seen": 2745237504
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.39512116835179e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.3721570364152926,
+      "tokens_seen": 2745368576
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.394318728935966e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.372144540428774,
+      "tokens_seen": 2745499648
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.3935162895201417e-05,
+      "loss": 1.1681,
+      "theoretical_loss": 3.372132045205838,
+      "tokens_seen": 2745630720
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.392713850104317e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.372119550746401,
+      "tokens_seen": 2745761792
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.391911410688493e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.37210705705038,
+      "tokens_seen": 2745892864
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.391108971272669e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.372094564117692,
+      "tokens_seen": 2746023936
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.390306531856845e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3720820719482534,
+      "tokens_seen": 2746155008
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.389504092441021e-05,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3720695805419814,
+      "tokens_seen": 2746286080
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.388701653025197e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3720570898987936,
+      "tokens_seen": 2746417152
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.387899213609373e-05,
+      "loss": 1.1891,
+      "theoretical_loss": 3.372044600018606,
+      "tokens_seen": 2746548224
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.387096774193548e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.372032110901337,
+      "tokens_seen": 2746679296
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.386294334777724e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.372019622546902,
+      "tokens_seen": 2746810368
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.3854918953619e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.3720071349552185,
+      "tokens_seen": 2746941440
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.384689455946076e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.371994648126204,
+      "tokens_seen": 2747072512
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.383887016530252e-05,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3719821620597754,
+      "tokens_seen": 2747203584
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.383084577114428e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.3719696767558496,
+      "tokens_seen": 2747334656
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.382282137698604e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.371957192214343,
+      "tokens_seen": 2747465728
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3814796982827795e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.3719447084351737,
+      "tokens_seen": 2747596800
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3806772588669555e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3719322254182584,
+      "tokens_seen": 2747727872
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3798748194511315e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3719197431635144,
+      "tokens_seen": 2747858944
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3790723800353074e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.3719072616708585,
+      "tokens_seen": 2747990016
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3782699406194834e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.371894780940208,
+      "tokens_seen": 2748121088
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3774675012036594e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.37188230097148,
+      "tokens_seen": 2748252160
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3766650617878354e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.3718698217645913,
+      "tokens_seen": 2748383232
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3758626223720114e-05,
+      "loss": 1.2069,
+      "theoretical_loss": 3.37185734331946,
+      "tokens_seen": 2748514304
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.375060182956187e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.371844865636002,
+      "tokens_seen": 2748645376
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.374257743540363e-05,
+      "loss": 1.1939,
+      "theoretical_loss": 3.3718323887141355,
+      "tokens_seen": 2748776448
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.373455304124539e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.371819912553777,
+      "tokens_seen": 2748907520
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.372652864708715e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.3718074371548443,
+      "tokens_seen": 2749038592
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.371850425292891e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.3717949625172547,
+      "tokens_seen": 2749169664
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.371047985877067e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.3717824886409247,
+      "tokens_seen": 2749300736
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.370245546461243e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3717700155257724,
+      "tokens_seen": 2749431808
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.369443107045418e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.3717575431717144,
+      "tokens_seen": 2749562880
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.368640667629594e-05,
+      "loss": 1.12,
+      "theoretical_loss": 3.3717450715786685,
+      "tokens_seen": 2749693952
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.36783822821377e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.3717326007465513,
+      "tokens_seen": 2749825024
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.367035788797946e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.371720130675281,
+      "tokens_seen": 2749956096
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.366233349382122e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.3717076613647743,
+      "tokens_seen": 2750087168
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.365430909966298e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.371695192814949,
+      "tokens_seen": 2750218240
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.364628470550474e-05,
+      "loss": 1.2138,
+      "theoretical_loss": 3.371682725025722,
+      "tokens_seen": 2750349312
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.363826031134649e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.3716702579970104,
+      "tokens_seen": 2750480384
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.363023591718825e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3716577917287323,
+      "tokens_seen": 2750611456
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.362221152303001e-05,
+      "loss": 1.178,
+      "theoretical_loss": 3.3716453262208046,
+      "tokens_seen": 2750742528
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.361418712887177e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.3716328614731452,
+      "tokens_seen": 2750873600
+    },
+    {
+      "epoch": 0.67,
+      "objective/train/docs_used": 923258,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.249371886253357,
+      "objective/train/theoretical_loss": 3.3716203974856707,
+      "objective/train/tokens_used": 1121463776,
+      "theoretical_loss": 3.3716203974856707,
+      "tokens_seen": 2751004672
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.360616273471353e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.3716203974856707,
+      "tokens_seen": 2751004672
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.359813834055529e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.3716079342582996,
+      "tokens_seen": 2751135744
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.359011394639705e-05,
+      "loss": 1.3097,
+      "theoretical_loss": 3.371595471790948,
+      "tokens_seen": 2751266816
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.358208955223881e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.3715830100835347,
+      "tokens_seen": 2751397888
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3574065158080565e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3715705491359764,
+      "tokens_seen": 2751528960
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3566040763922325e-05,
+      "loss": 1.1605,
+      "theoretical_loss": 3.3715580889481904,
+      "tokens_seen": 2751660032
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3558016369764085e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.371545629520095,
+      "tokens_seen": 2751791104
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3549991975605845e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.371533170851607,
+      "tokens_seen": 2751922176
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3541967581447604e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.371520712942644,
+      "tokens_seen": 2752053248
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3533943187289364e-05,
+      "loss": 1.1528,
+      "theoretical_loss": 3.3715082557931235,
+      "tokens_seen": 2752184320
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3525918793131124e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.3714957994029633,
+      "tokens_seen": 2752315392
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.351789439897288e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.371483343772081,
+      "tokens_seen": 2752446464
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.350987000481464e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3714708889003937,
+      "tokens_seen": 2752577536
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.35018456106564e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3714584347878196,
+      "tokens_seen": 2752708608
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.349382121649816e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.371445981434276,
+      "tokens_seen": 2752839680
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.348579682233992e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.37143352883968,
+      "tokens_seen": 2752970752
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.347777242818168e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.37142107700395,
+      "tokens_seen": 2753101824
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.346974803402344e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3714086259270033,
+      "tokens_seen": 2753232896
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.34617236398652e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.3713961756087576,
+      "tokens_seen": 2753363968
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.345369924570695e-05,
+      "loss": 1.1239,
+      "theoretical_loss": 3.371383726049131,
+      "tokens_seen": 2753495040
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.344567485154871e-05,
+      "loss": 1.1637,
+      "theoretical_loss": 3.37137127724804,
+      "tokens_seen": 2753626112
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.343765045739047e-05,
+      "loss": 1.1937,
+      "theoretical_loss": 3.371358829205403,
+      "tokens_seen": 2753757184
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.342962606323223e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.371346381921138,
+      "tokens_seen": 2753888256
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.342160166907399e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.371333935395162,
+      "tokens_seen": 2754019328
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.341357727491575e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.371321489627393,
+      "tokens_seen": 2754150400
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.340555288075751e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.371309044617749,
+      "tokens_seen": 2754281472
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.339752848659926e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3712966003661475,
+      "tokens_seen": 2754412544
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.338950409244102e-05,
+      "loss": 1.1833,
+      "theoretical_loss": 3.3712841568725067,
+      "tokens_seen": 2754543616
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.338147969828278e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.3712717141367436,
+      "tokens_seen": 2754674688
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.337345530412454e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.371259272158776,
+      "tokens_seen": 2754805760
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.33654309099663e-05,
+      "loss": 1.1832,
+      "theoretical_loss": 3.3712468309385226,
+      "tokens_seen": 2754936832
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.335740651580806e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.371234390475901,
+      "tokens_seen": 2755067904
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.334938212164982e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.371221950770828,
+      "tokens_seen": 2755198976
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3341357727491575e-05,
+      "loss": 1.2034,
+      "theoretical_loss": 3.3712095118232224,
+      "tokens_seen": 2755330048
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3711970736330015,
+      "tokens_seen": 2755461120
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3325308939175095e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.371184636200084,
+      "tokens_seen": 2755592192
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3317284545016855e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.371172199524387,
+      "tokens_seen": 2755723264
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3309260150858615e-05,
+      "loss": 1.2174,
+      "theoretical_loss": 3.3711597636058284,
+      "tokens_seen": 2755854336
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3301235756700375e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.371147328444326,
+      "tokens_seen": 2755985408
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3293211362542134e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.3711348940397987,
+      "tokens_seen": 2756116480
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3285186968383894e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.3711224603921632,
+      "tokens_seen": 2756247552
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.327716257422565e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.371110027501338,
+      "tokens_seen": 2756378624
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.326913818006741e-05,
+      "loss": 1.1549,
+      "theoretical_loss": 3.3710975953672415,
+      "tokens_seen": 2756509696
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.326111378590917e-05,
+      "loss": 1.1421,
+      "theoretical_loss": 3.3710851639897905,
+      "tokens_seen": 2756640768
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.325308939175093e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.371072733368904,
+      "tokens_seen": 2756771840
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.324506499759269e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.3710603035044997,
+      "tokens_seen": 2756902912
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.323704060343445e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3710478743964956,
+      "tokens_seen": 2757033984
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.322901620927621e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.3710354460448095,
+      "tokens_seen": 2757165056
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.322099181511796e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3710230184493595,
+      "tokens_seen": 2757296128
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.321296742095972e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.371010591610064,
+      "tokens_seen": 2757427200
+    },
+    {
+      "epoch": 0.67,
+      "objective/train/docs_used": 925397,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1923143863677979,
+      "objective/train/theoretical_loss": 3.3709981655268404,
+      "objective/train/tokens_used": 1128017376,
+      "theoretical_loss": 3.3709981655268404,
+      "tokens_seen": 2757558272
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.320494302680148e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3709981655268404,
+      "tokens_seen": 2757558272
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.319691863264324e-05,
+      "loss": 1.1668,
+      "theoretical_loss": 3.3709857401996075,
+      "tokens_seen": 2757689344
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3188894238485e-05,
+      "loss": 1.2602,
+      "theoretical_loss": 3.370973315628283,
+      "tokens_seen": 2757820416
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.318086984432676e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.3709608918127847,
+      "tokens_seen": 2757951488
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.317284545016851e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.3709484687530313,
+      "tokens_seen": 2758082560
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.316482105601027e-05,
+      "loss": 1.1903,
+      "theoretical_loss": 3.37093604644894,
+      "tokens_seen": 2758213632
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.315679666185203e-05,
+      "loss": 1.2751,
+      "theoretical_loss": 3.3709236249004304,
+      "tokens_seen": 2758344704
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.314877226769379e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.3709112041074194,
+      "tokens_seen": 2758475776
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.314074787353555e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.3708987840698255,
+      "tokens_seen": 2758606848
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.313272347937731e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.3708863647875673,
+      "tokens_seen": 2758737920
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3124699085219065e-05,
+      "loss": 1.2099,
+      "theoretical_loss": 3.370873946260562,
+      "tokens_seen": 2758868992
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3116674691060825e-05,
+      "loss": 1.2251,
+      "theoretical_loss": 3.370861528488729,
+      "tokens_seen": 2759000064
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3108650296902585e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.370849111471985,
+      "tokens_seen": 2759131136
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3100625902744345e-05,
+      "loss": 1.1602,
+      "theoretical_loss": 3.37083669521025,
+      "tokens_seen": 2759262208
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3092601508586105e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.370824279703441,
+      "tokens_seen": 2759393280
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.3084577114427865e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.3708118649514764,
+      "tokens_seen": 2759524352
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.307655272026962e-05,
+      "loss": 1.209,
+      "theoretical_loss": 3.370799450954275,
+      "tokens_seen": 2759655424
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.306852832611138e-05,
+      "loss": 1.1958,
+      "theoretical_loss": 3.3707870377117546,
+      "tokens_seen": 2759786496
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.306050393195314e-05,
+      "loss": 1.1792,
+      "theoretical_loss": 3.3707746252238335,
+      "tokens_seen": 2759917568
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.30524795377949e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3707622134904303,
+      "tokens_seen": 2760048640
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.304445514363666e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.370749802511463,
+      "tokens_seen": 2760179712
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.303643074947842e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.3707373922868498,
+      "tokens_seen": 2760310784
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.302840635532017e-05,
+      "loss": 1.2607,
+      "theoretical_loss": 3.3707249828165096,
+      "tokens_seen": 2760441856
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.302038196116193e-05,
+      "loss": 1.1827,
+      "theoretical_loss": 3.3707125741003603,
+      "tokens_seen": 2760572928
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.301235756700369e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.3707001661383202,
+      "tokens_seen": 2760704000
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.300433317284545e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3706877589303077,
+      "tokens_seen": 2760835072
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.299630877868721e-05,
+      "loss": 1.1603,
+      "theoretical_loss": 3.3706753524762414,
+      "tokens_seen": 2760966144
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.298828438452897e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.3706629467760396,
+      "tokens_seen": 2761097216
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.298025999037072e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3706505418296206,
+      "tokens_seen": 2761228288
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.297223559621248e-05,
+      "loss": 1.1797,
+      "theoretical_loss": 3.3706381376369032,
+      "tokens_seen": 2761359360
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.296421120205424e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3706257341978056,
+      "tokens_seen": 2761490432
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2956186807896e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3706133315122457,
+      "tokens_seen": 2761621504
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.294816241373776e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.370600929580143,
+      "tokens_seen": 2761752576
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.294013801957952e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.3705885284014157,
+      "tokens_seen": 2761883648
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.293211362542128e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.370576127975981,
+      "tokens_seen": 2762014720
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2924089231263036e-05,
+      "loss": 1.1337,
+      "theoretical_loss": 3.3705637283037593,
+      "tokens_seen": 2762145792
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2916064837104796e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.370551329384668,
+      "tokens_seen": 2762276864
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2908040442946556e-05,
+      "loss": 1.1424,
+      "theoretical_loss": 3.3705389312186256,
+      "tokens_seen": 2762407936
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2900016048788316e-05,
+      "loss": 1.1778,
+      "theoretical_loss": 3.3705265338055512,
+      "tokens_seen": 2762539008
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2891991654630076e-05,
+      "loss": 1.3322,
+      "theoretical_loss": 3.370514137145363,
+      "tokens_seen": 2762670080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2883967260471835e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3705017412379794,
+      "tokens_seen": 2762801152
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.2875942866313595e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3704893460833194,
+      "tokens_seen": 2762932224
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.286791847215535e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.3704769516813013,
+      "tokens_seen": 2763063296
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.285989407799711e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.370464558031844,
+      "tokens_seen": 2763194368
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.285186968383887e-05,
+      "loss": 1.2327,
+      "theoretical_loss": 3.3704521651348656,
+      "tokens_seen": 2763325440
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.284384528968063e-05,
+      "loss": 1.1065,
+      "theoretical_loss": 3.370439772990285,
+      "tokens_seen": 2763456512
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 3.283582089552239e-05,
+      "loss": 1.2013,
+      "theoretical_loss": 3.370427381598021,
+      "tokens_seen": 2763587584
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.282779650136415e-05,
+      "loss": 1.1838,
+      "theoretical_loss": 3.370414990957992,
+      "tokens_seen": 2763718656
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.281977210720591e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.3704026010701167,
+      "tokens_seen": 2763849728
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.281174771304767e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.3703902119343137,
+      "tokens_seen": 2763980800
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 927529,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.176634430885315,
+      "objective/train/theoretical_loss": 3.370377823550502,
+      "objective/train/tokens_used": 1134570976,
+      "theoretical_loss": 3.370377823550502,
+      "tokens_seen": 2764111872
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.280372331888942e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.370377823550502,
+      "tokens_seen": 2764111872
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.279569892473118e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3703654359186,
+      "tokens_seen": 2764242944
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.278767453057294e-05,
+      "loss": 1.16,
+      "theoretical_loss": 3.3703530490385267,
+      "tokens_seen": 2764374016
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.27796501364147e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.3703406629102,
+      "tokens_seen": 2764505088
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.277162574225646e-05,
+      "loss": 1.2037,
+      "theoretical_loss": 3.37032827753354,
+      "tokens_seen": 2764636160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.276360134809822e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3703158929084647,
+      "tokens_seen": 2764767232
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.275557695393998e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.3703035090348927,
+      "tokens_seen": 2764898304
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2747552559781734e-05,
+      "loss": 1.1566,
+      "theoretical_loss": 3.3702911259127433,
+      "tokens_seen": 2765029376
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2739528165623493e-05,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3702787435419346,
+      "tokens_seen": 2765160448
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.273150377146525e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.370266361922386,
+      "tokens_seen": 2765291520
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.272347937730701e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.3702539810540157,
+      "tokens_seen": 2765422592
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.271545498314877e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.370241600936743,
+      "tokens_seen": 2765553664
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.270743058899053e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.370229221570487,
+      "tokens_seen": 2765684736
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.269940619483229e-05,
+      "loss": 1.1939,
+      "theoretical_loss": 3.370216842955166,
+      "tokens_seen": 2765815808
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2691381800674046e-05,
+      "loss": 1.2078,
+      "theoretical_loss": 3.3702044650906995,
+      "tokens_seen": 2765946880
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2683357406515806e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3701920879770055,
+      "tokens_seen": 2766077952
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2675333012357566e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.370179711614003,
+      "tokens_seen": 2766209024
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2667308618199326e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.370167336001612,
+      "tokens_seen": 2766340096
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2659284224041086e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.37015496113975,
+      "tokens_seen": 2766471168
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2651259829882846e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.3701425870283366,
+      "tokens_seen": 2766602240
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2643235435724606e-05,
+      "loss": 1.1964,
+      "theoretical_loss": 3.370130213667291,
+      "tokens_seen": 2766733312
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2635211041566365e-05,
+      "loss": 1.1798,
+      "theoretical_loss": 3.370117841056532,
+      "tokens_seen": 2766864384
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.262718664740812e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.370105469195978,
+      "tokens_seen": 2766995456
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.261916225324988e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.370093098085549,
+      "tokens_seen": 2767126528
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.261113785909164e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.370080727725163,
+      "tokens_seen": 2767257600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.26031134649334e-05,
+      "loss": 1.222,
+      "theoretical_loss": 3.3700683581147395,
+      "tokens_seen": 2767388672
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.259508907077516e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3700559892541975,
+      "tokens_seen": 2767519744
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.258706467661692e-05,
+      "loss": 1.2529,
+      "theoretical_loss": 3.3700436211434557,
+      "tokens_seen": 2767650816
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.257904028245868e-05,
+      "loss": 1.2314,
+      "theoretical_loss": 3.3700312537824333,
+      "tokens_seen": 2767781888
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.257101588830043e-05,
+      "loss": 1.1719,
+      "theoretical_loss": 3.37001888717105,
+      "tokens_seen": 2767912960
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.256299149414219e-05,
+      "loss": 1.2722,
+      "theoretical_loss": 3.370006521309224,
+      "tokens_seen": 2768044032
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.255496709998395e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3699941561968743,
+      "tokens_seen": 2768175104
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.254694270582571e-05,
+      "loss": 1.1748,
+      "theoretical_loss": 3.369981791833921,
+      "tokens_seen": 2768306176
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.253891831166747e-05,
+      "loss": 1.1509,
+      "theoretical_loss": 3.3699694282202826,
+      "tokens_seen": 2768437248
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.253089391750923e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.3699570653558775,
+      "tokens_seen": 2768568320
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.252286952335099e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.369944703240626,
+      "tokens_seen": 2768699392
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.251484512919275e-05,
+      "loss": 1.1946,
+      "theoretical_loss": 3.369932341874447,
+      "tokens_seen": 2768830464
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2506820735034504e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.3699199812572593,
+      "tokens_seen": 2768961536
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2498796340876264e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.369907621388982,
+      "tokens_seen": 2769092608
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2490771946718023e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.3698952622695346,
+      "tokens_seen": 2769223680
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.248274755255978e-05,
+      "loss": 1.1128,
+      "theoretical_loss": 3.369882903898836,
+      "tokens_seen": 2769354752
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.247472315840154e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.3698705462768057,
+      "tokens_seen": 2769485824
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.24666987642433e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.3698581894033626,
+      "tokens_seen": 2769616896
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.245867437008506e-05,
+      "loss": 1.177,
+      "theoretical_loss": 3.3698458332784265,
+      "tokens_seen": 2769747968
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2450649975926816e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.369833477901916,
+      "tokens_seen": 2769879040
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2442625581768576e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3698211232737503,
+      "tokens_seen": 2770010112
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2434601187610336e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3698087693938494,
+      "tokens_seen": 2770141184
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2426576793452096e-05,
+      "loss": 1.2056,
+      "theoretical_loss": 3.3697964162621323,
+      "tokens_seen": 2770272256
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2418552399293856e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3697840638785177,
+      "tokens_seen": 2770403328
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2410528005135616e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.3697717122429256,
+      "tokens_seen": 2770534400
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 929566,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2751986980438232,
+      "objective/train/theoretical_loss": 3.3697593613552748,
+      "objective/train/tokens_used": 1141124576,
+      "theoretical_loss": 3.3697593613552748,
+      "tokens_seen": 2770665472
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2402503610977376e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.3697593613552748,
+      "tokens_seen": 2770665472
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.239447921681913e-05,
+      "loss": 1.1648,
+      "theoretical_loss": 3.3697470112154853,
+      "tokens_seen": 2770796544
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.238645482266089e-05,
+      "loss": 1.2067,
+      "theoretical_loss": 3.369734661823476,
+      "tokens_seen": 2770927616
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.237843042850265e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.369722313179166,
+      "tokens_seen": 2771058688
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.237040603434441e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.369709965282475,
+      "tokens_seen": 2771189760
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.236238164018617e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3696976181333227,
+      "tokens_seen": 2771320832
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.235435724602793e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.369685271731628,
+      "tokens_seen": 2771451904
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.234633285186969e-05,
+      "loss": 1.2151,
+      "theoretical_loss": 3.36967292607731,
+      "tokens_seen": 2771582976
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.233830845771145e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.369660581170289,
+      "tokens_seen": 2771714048
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.23302840635532e-05,
+      "loss": 1.1352,
+      "theoretical_loss": 3.369648237010484,
+      "tokens_seen": 2771845120
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.232225966939496e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.369635893597814,
+      "tokens_seen": 2771976192
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.231423527523672e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3696235509321992,
+      "tokens_seen": 2772107264
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.230621088107848e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.369611209013559,
+      "tokens_seen": 2772238336
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.229818648692024e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.369598867841812,
+      "tokens_seen": 2772369408
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2290162092762e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.3695865274168786,
+      "tokens_seen": 2772500480
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.228213769860376e-05,
+      "loss": 1.2825,
+      "theoretical_loss": 3.369574187738678,
+      "tokens_seen": 2772631552
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2274113304445514e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.36956184880713,
+      "tokens_seen": 2772762624
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2266088910287274e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3695495106221536,
+      "tokens_seen": 2772893696
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2258064516129034e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.3695371731836685,
+      "tokens_seen": 2773024768
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2250040121970793e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.369524836491595,
+      "tokens_seen": 2773155840
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2242015727812553e-05,
+      "loss": 1.1816,
+      "theoretical_loss": 3.3695125005458513,
+      "tokens_seen": 2773286912
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.223399133365431e-05,
+      "loss": 1.1419,
+      "theoretical_loss": 3.369500165346358,
+      "tokens_seen": 2773417984
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.222596693949607e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.3694878308930347,
+      "tokens_seen": 2773549056
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2217942545337826e-05,
+      "loss": 1.1673,
+      "theoretical_loss": 3.3694754971858,
+      "tokens_seen": 2773680128
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2209918151179586e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.369463164224575,
+      "tokens_seen": 2773811200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2201893757021346e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.369450832009278,
+      "tokens_seen": 2773942272
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2193869362863106e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3694385005398293,
+      "tokens_seen": 2774073344
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2185844968704866e-05,
+      "loss": 1.1855,
+      "theoretical_loss": 3.3694261698161485,
+      "tokens_seen": 2774204416
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2177820574546626e-05,
+      "loss": 1.1756,
+      "theoretical_loss": 3.3694138398381552,
+      "tokens_seen": 2774335488
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2169796180388386e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.369401510605769,
+      "tokens_seen": 2774466560
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2161771786230146e-05,
+      "loss": 1.0834,
+      "theoretical_loss": 3.36938918211891,
+      "tokens_seen": 2774597632
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.21537473920719e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3693768543774976,
+      "tokens_seen": 2774728704
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.214572299791366e-05,
+      "loss": 1.1705,
+      "theoretical_loss": 3.369364527381451,
+      "tokens_seen": 2774859776
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.213769860375542e-05,
+      "loss": 1.1217,
+      "theoretical_loss": 3.369352201130691,
+      "tokens_seen": 2774990848
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.212967420959718e-05,
+      "loss": 1.1432,
+      "theoretical_loss": 3.369339875625136,
+      "tokens_seen": 2775121920
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.212164981543894e-05,
+      "loss": 1.1601,
+      "theoretical_loss": 3.3693275508647074,
+      "tokens_seen": 2775252992
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.21136254212807e-05,
+      "loss": 1.2588,
+      "theoretical_loss": 3.3693152268493236,
+      "tokens_seen": 2775384064
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.210560102712246e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.369302903578905,
+      "tokens_seen": 2775515136
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.209757663296421e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.369290581053371,
+      "tokens_seen": 2775646208
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.208955223880597e-05,
+      "loss": 1.1704,
+      "theoretical_loss": 3.369278259272642,
+      "tokens_seen": 2775777280
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.208152784464773e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.3692659382366377,
+      "tokens_seen": 2775908352
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.207350345048949e-05,
+      "loss": 1.1775,
+      "theoretical_loss": 3.3692536179452772,
+      "tokens_seen": 2776039424
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.206547905633125e-05,
+      "loss": 1.1682,
+      "theoretical_loss": 3.369241298398481,
+      "tokens_seen": 2776170496
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.205745466217301e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.369228979596169,
+      "tokens_seen": 2776301568
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.204943026801477e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.369216661538261,
+      "tokens_seen": 2776432640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2041405873856524e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.369204344224676,
+      "tokens_seen": 2776563712
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2033381479698284e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.3691920276553353,
+      "tokens_seen": 2776694784
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2025357085540044e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.3691797118301583,
+      "tokens_seen": 2776825856
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2017332691381804e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3691673967490643,
+      "tokens_seen": 2776956928
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2009308297223564e-05,
+      "loss": 1.2822,
+      "theoretical_loss": 3.3691550824119734,
+      "tokens_seen": 2777088000
+    },
+    {
+      "epoch": 0.68,
+      "objective/train/docs_used": 931397,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2554059028625488,
+      "objective/train/theoretical_loss": 3.3691427688188065,
+      "objective/train/tokens_used": 1147678176,
+      "theoretical_loss": 3.3691427688188065,
+      "tokens_seen": 2777219072
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.2001283903065323e-05,
+      "loss": 1.2017,
+      "theoretical_loss": 3.3691427688188065,
+      "tokens_seen": 2777219072
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.199325950890708e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.369130455969483,
+      "tokens_seen": 2777350144
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.198523511474884e-05,
+      "loss": 1.1862,
+      "theoretical_loss": 3.369118143863922,
+      "tokens_seen": 2777481216
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1977210720590596e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.3691058325020444,
+      "tokens_seen": 2777612288
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1969186326432356e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.3690935218837703,
+      "tokens_seen": 2777743360
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1961161932274116e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.369081212009019,
+      "tokens_seen": 2777874432
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1953137538115876e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.369068902877711,
+      "tokens_seen": 2778005504
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1945113143957636e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.369056594489767,
+      "tokens_seen": 2778136576
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1937088749799396e-05,
+      "loss": 1.2139,
+      "theoretical_loss": 3.3690442868451056,
+      "tokens_seen": 2778267648
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.1929064355641156e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.3690319799436472,
+      "tokens_seen": 2778398720
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.192103996148291e-05,
+      "loss": 1.272,
+      "theoretical_loss": 3.3690196737853126,
+      "tokens_seen": 2778529792
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.191301556732467e-05,
+      "loss": 1.1672,
+      "theoretical_loss": 3.369007368370022,
+      "tokens_seen": 2778660864
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.190499117316643e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.368995063697694,
+      "tokens_seen": 2778791936
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.189696677900819e-05,
+      "loss": 1.1491,
+      "theoretical_loss": 3.3689827597682505,
+      "tokens_seen": 2778923008
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.188894238484995e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3689704565816103,
+      "tokens_seen": 2779054080
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.188091799069171e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3689581541376943,
+      "tokens_seen": 2779185152
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.187289359653347e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.368945852436422,
+      "tokens_seen": 2779316224
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.186486920237522e-05,
+      "loss": 1.1612,
+      "theoretical_loss": 3.3689335514777143,
+      "tokens_seen": 2779447296
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.185684480821698e-05,
+      "loss": 1.1661,
+      "theoretical_loss": 3.3689212512614906,
+      "tokens_seen": 2779578368
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.184882041405874e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3689089517876716,
+      "tokens_seen": 2779709440
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.18407960199005e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.368896653056177,
+      "tokens_seen": 2779840512
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.183277162574226e-05,
+      "loss": 1.1614,
+      "theoretical_loss": 3.3688843550669274,
+      "tokens_seen": 2779971584
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.182474723158402e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.3688720578198437,
+      "tokens_seen": 2780102656
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1816722837425774e-05,
+      "loss": 1.1696,
+      "theoretical_loss": 3.3688597613148445,
+      "tokens_seen": 2780233728
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1808698443267534e-05,
+      "loss": 1.252,
+      "theoretical_loss": 3.368847465551851,
+      "tokens_seen": 2780364800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1800674049109294e-05,
+      "loss": 1.1732,
+      "theoretical_loss": 3.3688351705307835,
+      "tokens_seen": 2780495872
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1792649654951054e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.368822876251562,
+      "tokens_seen": 2780626944
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1784625260792814e-05,
+      "loss": 1.1913,
+      "theoretical_loss": 3.368810582714107,
+      "tokens_seen": 2780758016
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1776600866634574e-05,
+      "loss": 1.1728,
+      "theoretical_loss": 3.3687982899183386,
+      "tokens_seen": 2780889088
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.176857647247633e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.368785997864177,
+      "tokens_seen": 2781020160
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.176055207831809e-05,
+      "loss": 1.2917,
+      "theoretical_loss": 3.368773706551543,
+      "tokens_seen": 2781151232
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.175252768415985e-05,
+      "loss": 1.1781,
+      "theoretical_loss": 3.3687614159803565,
+      "tokens_seen": 2781282304
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1744503290001607e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.368749126150538,
+      "tokens_seen": 2781413376
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1736478895843366e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.3687368370620074,
+      "tokens_seen": 2781544448
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1728454501685126e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3687245487146855,
+      "tokens_seen": 2781675520
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.172043010752688e-05,
+      "loss": 1.2653,
+      "theoretical_loss": 3.368712261108493,
+      "tokens_seen": 2781806592
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.171240571336864e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3686999742433494,
+      "tokens_seen": 2781937664
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.17043813192104e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.368687688119176,
+      "tokens_seen": 2782068736
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.169635692505216e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3686754027358923,
+      "tokens_seen": 2782199808
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.168833253089392e-05,
+      "loss": 1.1887,
+      "theoretical_loss": 3.3686631180934197,
+      "tokens_seen": 2782330880
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.168030813673568e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.368650834191678,
+      "tokens_seen": 2782461952
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.167228374257743e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.3686385510305876,
+      "tokens_seen": 2782593024
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.166425934841919e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.368626268610069,
+      "tokens_seen": 2782724096
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.165623495426095e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3686139869300433,
+      "tokens_seen": 2782855168
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.164821056010271e-05,
+      "loss": 1.182,
+      "theoretical_loss": 3.3686017059904305,
+      "tokens_seen": 2782986240
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.164018616594447e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.368589425791151,
+      "tokens_seen": 2783117312
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.163216177178623e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.3685771463321252,
+      "tokens_seen": 2783248384
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1624137377627985e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.368564867613274,
+      "tokens_seen": 2783379456
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1616112983469745e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.368552589634518,
+      "tokens_seen": 2783510528
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1608088589311505e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.368540312395777,
+      "tokens_seen": 2783641600
+    },
+    {
+      "epoch": 0.69,
+      "objective/train/docs_used": 933616,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2050771713256836,
+      "objective/train/theoretical_loss": 3.3685280358969725,
+      "objective/train/tokens_used": 1154231776,
+      "theoretical_loss": 3.3685280358969725,
+      "tokens_seen": 2783772672
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1600064195153265e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3685280358969725,
+      "tokens_seen": 2783772672
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1592039800995024e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.368515760138024,
+      "tokens_seen": 2783903744
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1584015406836784e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.368503485118853,
+      "tokens_seen": 2784034816
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1575991012678544e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.3684912108393803,
+      "tokens_seen": 2784165888
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.15679666185203e-05,
+      "loss": 1.1685,
+      "theoretical_loss": 3.3684789372995256,
+      "tokens_seen": 2784296960
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.155994222436206e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.36846666449921,
+      "tokens_seen": 2784428032
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.155191783020382e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3684543924383537,
+      "tokens_seen": 2784559104
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.154389343604558e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3684421211168782,
+      "tokens_seen": 2784690176
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.153586904188734e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.3684298505347035,
+      "tokens_seen": 2784821248
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.15278446477291e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3684175806917502,
+      "tokens_seen": 2784952320
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.151982025357086e-05,
+      "loss": 1.3177,
+      "theoretical_loss": 3.3684053115879395,
+      "tokens_seen": 2785083392
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.151179585941262e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3683930432231914,
+      "tokens_seen": 2785214464
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.150377146525437e-05,
+      "loss": 1.1994,
+      "theoretical_loss": 3.3683807755974273,
+      "tokens_seen": 2785345536
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.149574707109613e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.3683685087105677,
+      "tokens_seen": 2785476608
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.148772267693789e-05,
+      "loss": 1.3228,
+      "theoretical_loss": 3.368356242562533,
+      "tokens_seen": 2785607680
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.147969828277965e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.368343977153244,
+      "tokens_seen": 2785738752
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.147167388862141e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.368331712482622,
+      "tokens_seen": 2785869824
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.146364949446317e-05,
+      "loss": 1.2848,
+      "theoretical_loss": 3.3683194485505874,
+      "tokens_seen": 2786000896
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.145562510030493e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.368307185357061,
+      "tokens_seen": 2786131968
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.144760070614668e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.3682949229019634,
+      "tokens_seen": 2786263040
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.143957631198844e-05,
+      "loss": 1.3041,
+      "theoretical_loss": 3.3682826611852152,
+      "tokens_seen": 2786394112
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.14315519178302e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.368270400206738,
+      "tokens_seen": 2786525184
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.142352752367196e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.368258139966452,
+      "tokens_seen": 2786656256
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.141550312951372e-05,
+      "loss": 1.3078,
+      "theoretical_loss": 3.3682458804642783,
+      "tokens_seen": 2786787328
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.140747873535548e-05,
+      "loss": 1.327,
+      "theoretical_loss": 3.368233621700138,
+      "tokens_seen": 2786918400
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.139945434119724e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.3682213636739515,
+      "tokens_seen": 2787049472
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1391429947039e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.3682091063856396,
+      "tokens_seen": 2787180544
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1383405552880755e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.3681968498351234,
+      "tokens_seen": 2787311616
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1375381158722515e-05,
+      "loss": 1.1876,
+      "theoretical_loss": 3.368184594022324,
+      "tokens_seen": 2787442688
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1367356764564275e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.3681723389471623,
+      "tokens_seen": 2787573760
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1359332370406035e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.3681600846095585,
+      "tokens_seen": 2787704832
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1351307976247795e-05,
+      "loss": 1.2501,
+      "theoretical_loss": 3.368147831009434,
+      "tokens_seen": 2787835904
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1343283582089554e-05,
+      "loss": 1.3011,
+      "theoretical_loss": 3.3681355781467106,
+      "tokens_seen": 2787966976
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1335259187931314e-05,
+      "loss": 1.1746,
+      "theoretical_loss": 3.368123326021308,
+      "tokens_seen": 2788098048
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.132723479377307e-05,
+      "loss": 1.3585,
+      "theoretical_loss": 3.368111074633148,
+      "tokens_seen": 2788229120
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.131921039961483e-05,
+      "loss": 1.3221,
+      "theoretical_loss": 3.3680988239821508,
+      "tokens_seen": 2788360192
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.131118600545659e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.368086574068238,
+      "tokens_seen": 2788491264
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.130316161129835e-05,
+      "loss": 1.1839,
+      "theoretical_loss": 3.3680743248913303,
+      "tokens_seen": 2788622336
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.129513721714011e-05,
+      "loss": 1.2981,
+      "theoretical_loss": 3.3680620764513494,
+      "tokens_seen": 2788753408
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.128711282298187e-05,
+      "loss": 1.2832,
+      "theoretical_loss": 3.368049828748215,
+      "tokens_seen": 2788884480
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.127908842882363e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.3680375817818495,
+      "tokens_seen": 2789015552
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.127106403466538e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3680253355521734,
+      "tokens_seen": 2789146624
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.126303964050714e-05,
+      "loss": 1.3587,
+      "theoretical_loss": 3.3680130900591077,
+      "tokens_seen": 2789277696
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.12550152463489e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.368000845302573,
+      "tokens_seen": 2789408768
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.124699085219066e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.367988601282492,
+      "tokens_seen": 2789539840
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.123896645803242e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.367976357998784,
+      "tokens_seen": 2789670912
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.123094206387418e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.367964115451371,
+      "tokens_seen": 2789801984
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.122291766971594e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.367951873640174,
+      "tokens_seen": 2789933056
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.12148932755577e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.3679396325651147,
+      "tokens_seen": 2790064128
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.120686888139945e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.367927392226113,
+      "tokens_seen": 2790195200
+    },
+    {
+      "epoch": 0.69,
+      "objective/train/docs_used": 935680,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2257362604141235,
+      "objective/train/theoretical_loss": 3.367915152623091,
+      "objective/train/tokens_used": 1160785376,
+      "theoretical_loss": 3.367915152623091,
+      "tokens_seen": 2790326272
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.119884448724121e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.367915152623091,
+      "tokens_seen": 2790326272
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.119082009308297e-05,
+      "loss": 1.1929,
+      "theoretical_loss": 3.3679029137559695,
+      "tokens_seen": 2790457344
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.118279569892473e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.36789067562467,
+      "tokens_seen": 2790588416
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.117477130476649e-05,
+      "loss": 1.1848,
+      "theoretical_loss": 3.3678784382291136,
+      "tokens_seen": 2790719488
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.116674691060825e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3678662015692216,
+      "tokens_seen": 2790850560
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.115872251645001e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.3678539656449145,
+      "tokens_seen": 2790981632
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1150698122291765e-05,
+      "loss": 1.209,
+      "theoretical_loss": 3.3678417304561146,
+      "tokens_seen": 2791112704
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1142673728133525e-05,
+      "loss": 1.2985,
+      "theoretical_loss": 3.3678294960027424,
+      "tokens_seen": 2791243776
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1134649333975285e-05,
+      "loss": 1.1703,
+      "theoretical_loss": 3.36781726228472,
+      "tokens_seen": 2791374848
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1126624939817045e-05,
+      "loss": 1.1823,
+      "theoretical_loss": 3.3678050293019677,
+      "tokens_seen": 2791505920
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1118600545658805e-05,
+      "loss": 1.2637,
+      "theoretical_loss": 3.367792797054407,
+      "tokens_seen": 2791636992
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1110576151500565e-05,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3677805655419597,
+      "tokens_seen": 2791768064
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.1102551757342325e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.3677683347645466,
+      "tokens_seen": 2791899136
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.109452736318408e-05,
+      "loss": 1.1871,
+      "theoretical_loss": 3.3677561047220896,
+      "tokens_seen": 2792030208
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.108650296902584e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3677438754145093,
+      "tokens_seen": 2792161280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.10784785748676e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.3677316468417278,
+      "tokens_seen": 2792292352
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.107045418070936e-05,
+      "loss": 1.2979,
+      "theoretical_loss": 3.3677194190036657,
+      "tokens_seen": 2792423424
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.106242978655112e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.367707191900245,
+      "tokens_seen": 2792554496
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.105440539239288e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.3676949655313866,
+      "tokens_seen": 2792685568
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.104638099823464e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.3676827398970124,
+      "tokens_seen": 2792816640
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.10383566040764e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.367670514997043,
+      "tokens_seen": 2792947712
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.103033220991815e-05,
+      "loss": 1.3005,
+      "theoretical_loss": 3.3676582908314012,
+      "tokens_seen": 2793078784
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.102230781575991e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.367646067400007,
+      "tokens_seen": 2793209856
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.101428342160167e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.367633844702783,
+      "tokens_seen": 2793340928
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.100625902744343e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.36762162273965,
+      "tokens_seen": 2793472000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.099823463328519e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.3676094015105287,
+      "tokens_seen": 2793603072
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.099021023912695e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.3675971810153422,
+      "tokens_seen": 2793734144
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.098218584496871e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3675849612540114,
+      "tokens_seen": 2793865216
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.097416145081046e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.367572742226457,
+      "tokens_seen": 2793996288
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.096613705665222e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.367560523932602,
+      "tokens_seen": 2794127360
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.095811266249398e-05,
+      "loss": 1.2911,
+      "theoretical_loss": 3.367548306372367,
+      "tokens_seen": 2794258432
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.095008826833574e-05,
+      "loss": 1.2354,
+      "theoretical_loss": 3.367536089545673,
+      "tokens_seen": 2794389504
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.09420638741775e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3675238734524426,
+      "tokens_seen": 2794520576
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.093403948001926e-05,
+      "loss": 1.315,
+      "theoretical_loss": 3.367511658092597,
+      "tokens_seen": 2794651648
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.092601508586102e-05,
+      "loss": 1.2772,
+      "theoretical_loss": 3.3674994434660577,
+      "tokens_seen": 2794782720
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.091799069170278e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3674872295727463,
+      "tokens_seen": 2794913792
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0909966297544535e-05,
+      "loss": 1.3453,
+      "theoretical_loss": 3.367475016412584,
+      "tokens_seen": 2795044864
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0901941903386295e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3674628039854935,
+      "tokens_seen": 2795175936
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0893917509228055e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.3674505922913958,
+      "tokens_seen": 2795307008
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0885893115069815e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.3674383813302122,
+      "tokens_seen": 2795438080
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0877868720911575e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.3674261711018647,
+      "tokens_seen": 2795569152
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0869844326753335e-05,
+      "loss": 1.2268,
+      "theoretical_loss": 3.367413961606275,
+      "tokens_seen": 2795700224
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.0861819932595095e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3674017528433646,
+      "tokens_seen": 2795831296
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.085379553843685e-05,
+      "loss": 1.1771,
+      "theoretical_loss": 3.3673895448130553,
+      "tokens_seen": 2795962368
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.084577114427861e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3673773375152694,
+      "tokens_seen": 2796093440
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.083774675012037e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3673651309499273,
+      "tokens_seen": 2796224512
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.082972235596213e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.367352925116952,
+      "tokens_seen": 2796355584
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.082169796180389e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.367340720016264,
+      "tokens_seen": 2796486656
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 3.081367356764565e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.367328515647786,
+      "tokens_seen": 2796617728
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.080564917348741e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3673163120114395,
+      "tokens_seen": 2796748800
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 937796,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1570358276367188,
+      "objective/train/theoretical_loss": 3.3673041091071463,
+      "objective/train/tokens_used": 1167338976,
+      "theoretical_loss": 3.3673041091071463,
+      "tokens_seen": 2796879872
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.079762477932916e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3673041091071463,
+      "tokens_seen": 2796879872
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.078960038517092e-05,
+      "loss": 1.1994,
+      "theoretical_loss": 3.367291906934828,
+      "tokens_seen": 2797010944
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.078157599101268e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.367279705494407,
+      "tokens_seen": 2797142016
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.077355159685444e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.367267504785804,
+      "tokens_seen": 2797273088
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.07655272026962e-05,
+      "loss": 1.2013,
+      "theoretical_loss": 3.3672553048089413,
+      "tokens_seen": 2797404160
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.075750280853796e-05,
+      "loss": 1.2327,
+      "theoretical_loss": 3.367243105563741,
+      "tokens_seen": 2797535232
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.074947841437972e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3672309070501254,
+      "tokens_seen": 2797666304
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.074145402022148e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.3672187092680153,
+      "tokens_seen": 2797797376
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.073342962606323e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.367206512217333,
+      "tokens_seen": 2797928448
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.072540523190499e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.3671943158980007,
+      "tokens_seen": 2798059520
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.071738083774675e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.36718212030994,
+      "tokens_seen": 2798190592
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.070935644358851e-05,
+      "loss": 1.1779,
+      "theoretical_loss": 3.367169925453073,
+      "tokens_seen": 2798321664
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.070133204943027e-05,
+      "loss": 1.2819,
+      "theoretical_loss": 3.3671577313273207,
+      "tokens_seen": 2798452736
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.069330765527203e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.3671455379326063,
+      "tokens_seen": 2798583808
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.068528326111379e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.367133345268851,
+      "tokens_seen": 2798714880
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0677258866955545e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.367121153335977,
+      "tokens_seen": 2798845952
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0669234472797305e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3671089621339063,
+      "tokens_seen": 2798977024
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0661210078639065e-05,
+      "loss": 1.2229,
+      "theoretical_loss": 3.3670967716625606,
+      "tokens_seen": 2799108096
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0653185684480825e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.3670845819218624,
+      "tokens_seen": 2799239168
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0645161290322585e-05,
+      "loss": 1.2777,
+      "theoretical_loss": 3.367072392911733,
+      "tokens_seen": 2799370240
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0637136896164345e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.367060204632095,
+      "tokens_seen": 2799501312
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0629112502006105e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.3670480170828703,
+      "tokens_seen": 2799632384
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.062108810784786e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3670358302639807,
+      "tokens_seen": 2799763456
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.061306371368962e-05,
+      "loss": 1.2026,
+      "theoretical_loss": 3.3670236441753483,
+      "tokens_seen": 2799894528
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.060503931953138e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3670114588168953,
+      "tokens_seen": 2800025600
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.059701492537314e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3669992741885437,
+      "tokens_seen": 2800156672
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.05889905312149e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3669870902902157,
+      "tokens_seen": 2800287744
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.058096613705666e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.3669749071218336,
+      "tokens_seen": 2800418816
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.057294174289842e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3669627246833187,
+      "tokens_seen": 2800549888
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.056491734874018e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.366950542974594,
+      "tokens_seen": 2800680960
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.055689295458193e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.366938361995581,
+      "tokens_seen": 2800812032
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.054886856042369e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.366926181746202,
+      "tokens_seen": 2800943104
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.054084416626545e-05,
+      "loss": 1.1791,
+      "theoretical_loss": 3.3669140022263795,
+      "tokens_seen": 2801074176
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.053281977210721e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3669018234360353,
+      "tokens_seen": 2801205248
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.052479537794897e-05,
+      "loss": 1.1658,
+      "theoretical_loss": 3.3668896453750916,
+      "tokens_seen": 2801336320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0516770983790727e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3668774680434708,
+      "tokens_seen": 2801467392
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0508746589632486e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.366865291441095,
+      "tokens_seen": 2801598464
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.050072219547424e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.3668531155678862,
+      "tokens_seen": 2801729536
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0492697801316e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.366840940423767,
+      "tokens_seen": 2801860608
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.048467340715776e-05,
+      "loss": 1.3,
+      "theoretical_loss": 3.3668287660086595,
+      "tokens_seen": 2801991680
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.047664901299952e-05,
+      "loss": 1.2735,
+      "theoretical_loss": 3.3668165923224858,
+      "tokens_seen": 2802122752
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.046862461884128e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.3668044193651685,
+      "tokens_seen": 2802253824
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.046060022468304e-05,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3667922471366296,
+      "tokens_seen": 2802384896
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.04525758305248e-05,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3667800756367914,
+      "tokens_seen": 2802515968
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.044455143636656e-05,
+      "loss": 1.1288,
+      "theoretical_loss": 3.366767904865576,
+      "tokens_seen": 2802647040
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0436527042208312e-05,
+      "loss": 1.2079,
+      "theoretical_loss": 3.366755734822906,
+      "tokens_seen": 2802778112
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0428502648050072e-05,
+      "loss": 1.1811,
+      "theoretical_loss": 3.366743565508704,
+      "tokens_seen": 2802909184
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0420478253891832e-05,
+      "loss": 1.1924,
+      "theoretical_loss": 3.3667313969228916,
+      "tokens_seen": 2803040256
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0412453859733592e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.366719229065392,
+      "tokens_seen": 2803171328
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.040442946557535e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.366707061936127,
+      "tokens_seen": 2803302400
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 939819,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1384387016296387,
+      "objective/train/theoretical_loss": 3.366694895535019,
+      "objective/train/tokens_used": 1173892576,
+      "theoretical_loss": 3.366694895535019,
+      "tokens_seen": 2803433472
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.039640507141711e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.366694895535019,
+      "tokens_seen": 2803433472
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.038838067725887e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.3666827298619904,
+      "tokens_seen": 2803564544
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0380356283100625e-05,
+      "loss": 1.2094,
+      "theoretical_loss": 3.3666705649169635,
+      "tokens_seen": 2803695616
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0372331888942384e-05,
+      "loss": 1.1713,
+      "theoretical_loss": 3.366658400699861,
+      "tokens_seen": 2803826688
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0364307494784144e-05,
+      "loss": 1.3026,
+      "theoretical_loss": 3.366646237210605,
+      "tokens_seen": 2803957760
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0356283100625904e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3666340744491183,
+      "tokens_seen": 2804088832
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0348258706467664e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.366621912415323,
+      "tokens_seen": 2804219904
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0340234312309424e-05,
+      "loss": 1.1819,
+      "theoretical_loss": 3.366609751109142,
+      "tokens_seen": 2804350976
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0332209918151184e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3665975905304975,
+      "tokens_seen": 2804482048
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0324185523992937e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.3665854306793115,
+      "tokens_seen": 2804613120
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0316161129834697e-05,
+      "loss": 1.1505,
+      "theoretical_loss": 3.3665732715555072,
+      "tokens_seen": 2804744192
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0308136735676457e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.366561113159007,
+      "tokens_seen": 2804875264
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0300112341518217e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.3665489554897325,
+      "tokens_seen": 2805006336
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0292087947359977e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.3665367985476076,
+      "tokens_seen": 2805137408
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0284063553201737e-05,
+      "loss": 1.1763,
+      "theoretical_loss": 3.3665246423325543,
+      "tokens_seen": 2805268480
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0276039159043497e-05,
+      "loss": 1.2904,
+      "theoretical_loss": 3.3665124868444947,
+      "tokens_seen": 2805399552
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0268014764885256e-05,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3665003320833518,
+      "tokens_seen": 2805530624
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.025999037072701e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.366488178049048,
+      "tokens_seen": 2805661696
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.025196597656877e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.366476024741506,
+      "tokens_seen": 2805792768
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.024394158241053e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3664638721606486,
+      "tokens_seen": 2805923840
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.023591718825229e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3664517203063977,
+      "tokens_seen": 2806054912
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.022789279409405e-05,
+      "loss": 1.2992,
+      "theoretical_loss": 3.3664395691786764,
+      "tokens_seen": 2806185984
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.021986839993581e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3664274187774077,
+      "tokens_seen": 2806317056
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.021184400577757e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3664152691025135,
+      "tokens_seen": 2806448128
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0203819611619322e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.366403120153917,
+      "tokens_seen": 2806579200
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0195795217461082e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.366390971931541,
+      "tokens_seen": 2806710272
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0187770823302842e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3663788244353072,
+      "tokens_seen": 2806841344
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0179746429144602e-05,
+      "loss": 1.2985,
+      "theoretical_loss": 3.3663666776651393,
+      "tokens_seen": 2806972416
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0171722034986362e-05,
+      "loss": 1.3062,
+      "theoretical_loss": 3.3663545316209595,
+      "tokens_seen": 2807103488
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0163697640828122e-05,
+      "loss": 1.2438,
+      "theoretical_loss": 3.3663423863026907,
+      "tokens_seen": 2807234560
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0155673246669878e-05,
+      "loss": 1.1152,
+      "theoretical_loss": 3.366330241710256,
+      "tokens_seen": 2807365632
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0147648852511635e-05,
+      "loss": 1.1252,
+      "theoretical_loss": 3.3663180978435774,
+      "tokens_seen": 2807496704
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0139624458353395e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3663059547025775,
+      "tokens_seen": 2807627776
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0131600064195155e-05,
+      "loss": 1.1732,
+      "theoretical_loss": 3.36629381228718,
+      "tokens_seen": 2807758848
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0123575670036914e-05,
+      "loss": 1.207,
+      "theoretical_loss": 3.3662816705973073,
+      "tokens_seen": 2807889920
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0115551275878674e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.366269529632882,
+      "tokens_seen": 2808020992
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.010752688172043e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3662573893938266,
+      "tokens_seen": 2808152064
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.009950248756219e-05,
+      "loss": 1.1596,
+      "theoretical_loss": 3.366245249880065,
+      "tokens_seen": 2808283136
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.009147809340395e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.3662331110915185,
+      "tokens_seen": 2808414208
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0083453699245707e-05,
+      "loss": 1.1857,
+      "theoretical_loss": 3.366220973028111,
+      "tokens_seen": 2808545280
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0075429305087467e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.3662088356897657,
+      "tokens_seen": 2808676352
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0067404910929227e-05,
+      "loss": 1.2868,
+      "theoretical_loss": 3.3661966990764043,
+      "tokens_seen": 2808807424
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0059380516770984e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.36618456318795,
+      "tokens_seen": 2808938496
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0051356122612743e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.366172428024326,
+      "tokens_seen": 2809069568
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0043331728454503e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.3661602935854553,
+      "tokens_seen": 2809200640
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0035307334296263e-05,
+      "loss": 1.282,
+      "theoretical_loss": 3.3661481598712606,
+      "tokens_seen": 2809331712
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.002728294013802e-05,
+      "loss": 1.2048,
+      "theoretical_loss": 3.366136026881665,
+      "tokens_seen": 2809462784
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.001925854597978e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3661238946165906,
+      "tokens_seen": 2809593856
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0011234151821536e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.3661117630759616,
+      "tokens_seen": 2809724928
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 3.0003209757663296e-05,
+      "loss": 1.1873,
+      "theoretical_loss": 3.3660996322597,
+      "tokens_seen": 2809856000
+    },
+    {
+      "epoch": 0.7,
+      "objective/train/docs_used": 942090,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2293963432312012,
+      "objective/train/theoretical_loss": 3.366087502167729,
+      "objective/train/tokens_used": 1180446176,
+      "theoretical_loss": 3.366087502167729,
+      "tokens_seen": 2809987072
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9995185363505056e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.366087502167729,
+      "tokens_seen": 2809987072
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9987160969346816e-05,
+      "loss": 1.1614,
+      "theoretical_loss": 3.3660753727999717,
+      "tokens_seen": 2810118144
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9979136575188576e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.3660632441563507,
+      "tokens_seen": 2810249216
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9971112181030336e-05,
+      "loss": 1.2811,
+      "theoretical_loss": 3.36605111623679,
+      "tokens_seen": 2810380288
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.996308778687209e-05,
+      "loss": 1.2693,
+      "theoretical_loss": 3.3660389890412117,
+      "tokens_seen": 2810511360
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.995506339271385e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.3660268625695386,
+      "tokens_seen": 2810642432
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.994703899855561e-05,
+      "loss": 1.3128,
+      "theoretical_loss": 3.3660147368216946,
+      "tokens_seen": 2810773504
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.993901460439737e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.3660026117976027,
+      "tokens_seen": 2810904576
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.993099021023913e-05,
+      "loss": 1.2355,
+      "theoretical_loss": 3.365990487497185,
+      "tokens_seen": 2811035648
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.992296581608089e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.365978363920365,
+      "tokens_seen": 2811166720
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.991494142192265e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.3659662410670665,
+      "tokens_seen": 2811297792
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.99069170277644e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.365954118937212,
+      "tokens_seen": 2811428864
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.989889263360616e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.365941997530724,
+      "tokens_seen": 2811559936
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.989086823944792e-05,
+      "loss": 1.174,
+      "theoretical_loss": 3.365929876847527,
+      "tokens_seen": 2811691008
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.988284384528968e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.365917756887543,
+      "tokens_seen": 2811822080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.987481945113144e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3659056376506955,
+      "tokens_seen": 2811953152
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.98667950569732e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.365893519136908,
+      "tokens_seen": 2812084224
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.985877066281496e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.365881401346103,
+      "tokens_seen": 2812215296
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9850746268656714e-05,
+      "loss": 1.1801,
+      "theoretical_loss": 3.3658692842782036,
+      "tokens_seen": 2812346368
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9842721874498474e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.365857167933134,
+      "tokens_seen": 2812477440
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9834697480340234e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.3658450523108168,
+      "tokens_seen": 2812608512
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9826673086181994e-05,
+      "loss": 1.1396,
+      "theoretical_loss": 3.365832937411175,
+      "tokens_seen": 2812739584
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9818648692023754e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.365820823234132,
+      "tokens_seen": 2812870656
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9810624297865514e-05,
+      "loss": 1.2476,
+      "theoretical_loss": 3.365808709779611,
+      "tokens_seen": 2813001728
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 2.9802599903707273e-05,
+      "loss": 1.2782,
+      "theoretical_loss": 3.3657965970475354,
+      "tokens_seen": 2813132800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9794575509549033e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3657844850378282,
+      "tokens_seen": 2813263872
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9786551115390786e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3657723737504126,
+      "tokens_seen": 2813394944
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9778526721232546e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3657602631852126,
+      "tokens_seen": 2813526016
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9770502327074306e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.3657481533421505,
+      "tokens_seen": 2813657088
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9762477932916066e-05,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3657360442211504,
+      "tokens_seen": 2813788160
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9754453538757826e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3657239358221354,
+      "tokens_seen": 2813919232
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9746429144599586e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.3657118281450282,
+      "tokens_seen": 2814050304
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9738404750441346e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.365699721189753,
+      "tokens_seen": 2814181376
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.97303803562831e-05,
+      "loss": 1.1606,
+      "theoretical_loss": 3.365687614956233,
+      "tokens_seen": 2814312448
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.972235596212486e-05,
+      "loss": 1.3036,
+      "theoretical_loss": 3.365675509444391,
+      "tokens_seen": 2814443520
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.971433156796662e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.3656634046541507,
+      "tokens_seen": 2814574592
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.970630717380838e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.365651300585436,
+      "tokens_seen": 2814705664
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.969828277965014e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3656391972381696,
+      "tokens_seen": 2814836736
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.96902583854919e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.365627094612275,
+      "tokens_seen": 2814967808
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.968223399133366e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3656149927076755,
+      "tokens_seen": 2815098880
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.967420959717541e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3656028915242953,
+      "tokens_seen": 2815229952
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.966618520301717e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.365590791062057,
+      "tokens_seen": 2815361024
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.965816080885893e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3655786913208843,
+      "tokens_seen": 2815492096
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.965013641470069e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.365566592300701,
+      "tokens_seen": 2815623168
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.964211202054245e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.3655544940014304,
+      "tokens_seen": 2815754240
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.963408762638421e-05,
+      "loss": 1.3705,
+      "theoretical_loss": 3.3655423964229954,
+      "tokens_seen": 2815885312
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.962606323222597e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3655302995653207,
+      "tokens_seen": 2816016384
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.961803883806773e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.3655182034283286,
+      "tokens_seen": 2816147456
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9610014443909484e-05,
+      "loss": 1.2375,
+      "theoretical_loss": 3.3655061080119433,
+      "tokens_seen": 2816278528
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9601990049751244e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.365494013316088,
+      "tokens_seen": 2816409600
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 944328,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2537761926651,
+      "objective/train/theoretical_loss": 3.3654819193406866,
+      "objective/train/tokens_used": 1186999776,
+      "theoretical_loss": 3.3654819193406866,
+      "tokens_seen": 2816540672
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9593965655593004e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.3654819193406866,
+      "tokens_seen": 2816540672
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9585941261434764e-05,
+      "loss": 1.3084,
+      "theoretical_loss": 3.3654698260856626,
+      "tokens_seen": 2816671744
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9577916867276524e-05,
+      "loss": 1.2808,
+      "theoretical_loss": 3.3654577335509392,
+      "tokens_seen": 2816802816
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9569892473118284e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3654456417364407,
+      "tokens_seen": 2816933888
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9561868078960044e-05,
+      "loss": 1.2883,
+      "theoretical_loss": 3.36543355064209,
+      "tokens_seen": 2817064960
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9553843684801797e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.365421460267811,
+      "tokens_seen": 2817196032
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9545819290643557e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.365409370613527,
+      "tokens_seen": 2817327104
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9537794896485316e-05,
+      "loss": 1.2766,
+      "theoretical_loss": 3.3653972816791624,
+      "tokens_seen": 2817458176
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9529770502327076e-05,
+      "loss": 1.3001,
+      "theoretical_loss": 3.3653851934646397,
+      "tokens_seen": 2817589248
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9521746108168836e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3653731059698835,
+      "tokens_seen": 2817720320
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9513721714010596e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.3653610191948173,
+      "tokens_seen": 2817851392
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9505697319852356e-05,
+      "loss": 1.3106,
+      "theoretical_loss": 3.3653489331393645,
+      "tokens_seen": 2817982464
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9497672925694113e-05,
+      "loss": 1.3088,
+      "theoretical_loss": 3.365336847803449,
+      "tokens_seen": 2818113536
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.948964853153587e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3653247631869943,
+      "tokens_seen": 2818244608
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.948162413737763e-05,
+      "loss": 1.2887,
+      "theoretical_loss": 3.365312679289924,
+      "tokens_seen": 2818375680
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.947359974321939e-05,
+      "loss": 1.3421,
+      "theoretical_loss": 3.3653005961121627,
+      "tokens_seen": 2818506752
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.946557534906115e-05,
+      "loss": 1.2726,
+      "theoretical_loss": 3.365288513653633,
+      "tokens_seen": 2818637824
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.945755095490291e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3652764319142596,
+      "tokens_seen": 2818768896
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9449526560744665e-05,
+      "loss": 1.3153,
+      "theoretical_loss": 3.3652643508939653,
+      "tokens_seen": 2818899968
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9441502166586425e-05,
+      "loss": 1.2911,
+      "theoretical_loss": 3.3652522705926744,
+      "tokens_seen": 2819031040
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.943347777242818e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.3652401910103107,
+      "tokens_seen": 2819162112
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.942545337826994e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.365228112146798,
+      "tokens_seen": 2819293184
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.94174289841117e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.36521603400206,
+      "tokens_seen": 2819424256
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.940940458995346e-05,
+      "loss": 1.2241,
+      "theoretical_loss": 3.365203956576021,
+      "tokens_seen": 2819555328
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9401380195795218e-05,
+      "loss": 1.198,
+      "theoretical_loss": 3.365191879868604,
+      "tokens_seen": 2819686400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9393355801636978e-05,
+      "loss": 1.3067,
+      "theoretical_loss": 3.365179803879733,
+      "tokens_seen": 2819817472
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9385331407478738e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3651677286093324,
+      "tokens_seen": 2819948544
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9377307013320494e-05,
+      "loss": 1.1861,
+      "theoretical_loss": 3.3651556540573258,
+      "tokens_seen": 2820079616
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9369282619162254e-05,
+      "loss": 1.299,
+      "theoretical_loss": 3.365143580223637,
+      "tokens_seen": 2820210688
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9361258225004014e-05,
+      "loss": 1.2349,
+      "theoretical_loss": 3.365131507108189,
+      "tokens_seen": 2820341760
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.935323383084577e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.3651194347109077,
+      "tokens_seen": 2820472832
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.934520943668753e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3651073630317154,
+      "tokens_seen": 2820603904
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.933718504252929e-05,
+      "loss": 1.2809,
+      "theoretical_loss": 3.365095292070537,
+      "tokens_seen": 2820734976
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.932916064837105e-05,
+      "loss": 1.2337,
+      "theoretical_loss": 3.3650832218272955,
+      "tokens_seen": 2820866048
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.932113625421281e-05,
+      "loss": 1.2835,
+      "theoretical_loss": 3.3650711523019154,
+      "tokens_seen": 2820997120
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9313111860054567e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.3650590834943204,
+      "tokens_seen": 2821128192
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9305087465896323e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3650470154044347,
+      "tokens_seen": 2821259264
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9297063071738083e-05,
+      "loss": 1.3084,
+      "theoretical_loss": 3.365034948032182,
+      "tokens_seen": 2821390336
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9289038677579843e-05,
+      "loss": 1.1642,
+      "theoretical_loss": 3.365022881377487,
+      "tokens_seen": 2821521408
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9281014283421603e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.365010815440273,
+      "tokens_seen": 2821652480
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9272989889263363e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.3649987502204644,
+      "tokens_seen": 2821783552
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9264965495105123e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3649866857179846,
+      "tokens_seen": 2821914624
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9256941100946876e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.3649746219327583,
+      "tokens_seen": 2822045696
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9248916706788636e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3649625588647094,
+      "tokens_seen": 2822176768
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9240892312630396e-05,
+      "loss": 1.1751,
+      "theoretical_loss": 3.3649504965137615,
+      "tokens_seen": 2822307840
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9232867918472156e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3649384348798392,
+      "tokens_seen": 2822438912
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9224843524313916e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.3649263739628665,
+      "tokens_seen": 2822569984
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9216819130155675e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3649143137627675,
+      "tokens_seen": 2822701056
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9208794735997435e-05,
+      "loss": 1.1846,
+      "theoretical_loss": 3.364902254279466,
+      "tokens_seen": 2822832128
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.920077034183919e-05,
+      "loss": 1.3479,
+      "theoretical_loss": 3.3648901955128867,
+      "tokens_seen": 2822963200
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 946526,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2754766941070557,
+      "objective/train/theoretical_loss": 3.364878137462953,
+      "objective/train/tokens_used": 1193553376,
+      "theoretical_loss": 3.364878137462953,
+      "tokens_seen": 2823094272
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.919274594768095e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.364878137462953,
+      "tokens_seen": 2823094272
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9184721553522708e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3648660801295893,
+      "tokens_seen": 2823225344
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9176697159364468e-05,
+      "loss": 1.26,
+      "theoretical_loss": 3.3648540235127204,
+      "tokens_seen": 2823356416
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9168672765206228e-05,
+      "loss": 1.2745,
+      "theoretical_loss": 3.3648419676122696,
+      "tokens_seen": 2823487488
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9160648371047988e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.3648299124281618,
+      "tokens_seen": 2823618560
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9152623976889748e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.36481785796032,
+      "tokens_seen": 2823749632
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9144599582731508e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.36480580420867,
+      "tokens_seen": 2823880704
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.913657518857326e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.3647937511731345,
+      "tokens_seen": 2824011776
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.912855079441502e-05,
+      "loss": 1.2626,
+      "theoretical_loss": 3.3647816988536388,
+      "tokens_seen": 2824142848
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.912052640025678e-05,
+      "loss": 1.3434,
+      "theoretical_loss": 3.3647696472501067,
+      "tokens_seen": 2824273920
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.911250200609854e-05,
+      "loss": 1.2856,
+      "theoretical_loss": 3.3647575963624625,
+      "tokens_seen": 2824404992
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.91044776119403e-05,
+      "loss": 1.2778,
+      "theoretical_loss": 3.36474554619063,
+      "tokens_seen": 2824536064
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.909645321778206e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.3647334967345346,
+      "tokens_seen": 2824667136
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.908842882362382e-05,
+      "loss": 1.3164,
+      "theoretical_loss": 3.3647214479940994,
+      "tokens_seen": 2824798208
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9080404429465574e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.3647093999692497,
+      "tokens_seen": 2824929280
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9072380035307333e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.364697352659909,
+      "tokens_seen": 2825060352
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9064355641149093e-05,
+      "loss": 1.2361,
+      "theoretical_loss": 3.364685306066002,
+      "tokens_seen": 2825191424
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9056331246990853e-05,
+      "loss": 1.2623,
+      "theoretical_loss": 3.3646732601874527,
+      "tokens_seen": 2825322496
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9048306852832613e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.364661215024186,
+      "tokens_seen": 2825453568
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9040282458674373e-05,
+      "loss": 1.3098,
+      "theoretical_loss": 3.3646491705761257,
+      "tokens_seen": 2825584640
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9032258064516133e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.364637126843196,
+      "tokens_seen": 2825715712
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9024233670357893e-05,
+      "loss": 1.2135,
+      "theoretical_loss": 3.364625083825322,
+      "tokens_seen": 2825846784
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9016209276199646e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.3646130415224276,
+      "tokens_seen": 2825977856
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9008184882041406e-05,
+      "loss": 1.2886,
+      "theoretical_loss": 3.3646009999344377,
+      "tokens_seen": 2826108928
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.9000160487883166e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.364588959061276,
+      "tokens_seen": 2826240000
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8992136093724926e-05,
+      "loss": 1.2668,
+      "theoretical_loss": 3.364576918902867,
+      "tokens_seen": 2826371072
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8984111699566686e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.364564879459136,
+      "tokens_seen": 2826502144
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8976087305408446e-05,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3645528407300063,
+      "tokens_seen": 2826633216
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8968062911250205e-05,
+      "loss": 1.2434,
+      "theoretical_loss": 3.364540802715403,
+      "tokens_seen": 2826764288
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.896003851709196e-05,
+      "loss": 1.2461,
+      "theoretical_loss": 3.3645287654152503,
+      "tokens_seen": 2826895360
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.895201412293372e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3645167288294733,
+      "tokens_seen": 2827026432
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.894398972877548e-05,
+      "loss": 1.2598,
+      "theoretical_loss": 3.364504692957995,
+      "tokens_seen": 2827157504
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8935965334617238e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.364492657800742,
+      "tokens_seen": 2827288576
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8927940940458998e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.364480623357637,
+      "tokens_seen": 2827419648
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8919916546300758e-05,
+      "loss": 1.3083,
+      "theoretical_loss": 3.3644685896286055,
+      "tokens_seen": 2827550720
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8911892152142518e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.3644565566135713,
+      "tokens_seen": 2827681792
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.890386775798427e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.36444452431246,
+      "tokens_seen": 2827812864
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.889584336382603e-05,
+      "loss": 1.3332,
+      "theoretical_loss": 3.364432492725195,
+      "tokens_seen": 2827943936
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.888781896966779e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.3644204618517017,
+      "tokens_seen": 2828075008
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.887979457550955e-05,
+      "loss": 1.1787,
+      "theoretical_loss": 3.3644084316919045,
+      "tokens_seen": 2828206080
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.887177018135131e-05,
+      "loss": 1.3015,
+      "theoretical_loss": 3.3643964022457276,
+      "tokens_seen": 2828337152
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.886374578719307e-05,
+      "loss": 1.1512,
+      "theoretical_loss": 3.364384373513096,
+      "tokens_seen": 2828468224
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.885572139303483e-05,
+      "loss": 1.2802,
+      "theoretical_loss": 3.3643723454939343,
+      "tokens_seen": 2828599296
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.884769699887659e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.3643603181881665,
+      "tokens_seen": 2828730368
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8839672604718344e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.364348291595718,
+      "tokens_seen": 2828861440
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8831648210560103e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3643362657165135,
+      "tokens_seen": 2828992512
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8823623816401863e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.364324240550477,
+      "tokens_seen": 2829123584
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8815599422243623e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3643122160975336,
+      "tokens_seen": 2829254656
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8807575028085383e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.364300192357608,
+      "tokens_seen": 2829385728
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.8799550633927143e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.364288169330625,
+      "tokens_seen": 2829516800
+    },
+    {
+      "epoch": 0.71,
+      "objective/train/docs_used": 948624,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3058116436004639,
+      "objective/train/theoretical_loss": 3.3642761470165086,
+      "objective/train/tokens_used": 1200106976,
+      "theoretical_loss": 3.3642761470165086,
+      "tokens_seen": 2829647872
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.87915262397689e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3642761470165086,
+      "tokens_seen": 2829647872
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8783501845610656e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.364264125415184,
+      "tokens_seen": 2829778944
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8775477451452416e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.3642521045265763,
+      "tokens_seen": 2829910016
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8767453057294176e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3642400843506097,
+      "tokens_seen": 2830041088
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8759428663135936e-05,
+      "loss": 1.3076,
+      "theoretical_loss": 3.364228064887209,
+      "tokens_seen": 2830172160
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8751404268977696e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.364216046136299,
+      "tokens_seen": 2830303232
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8743379874819452e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.364204028097805,
+      "tokens_seen": 2830434304
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8735355480661212e-05,
+      "loss": 1.2644,
+      "theoretical_loss": 3.364192010771651,
+      "tokens_seen": 2830565376
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.872733108650297e-05,
+      "loss": 1.3009,
+      "theoretical_loss": 3.3641799941577624,
+      "tokens_seen": 2830696448
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.871930669234473e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3641679782560634,
+      "tokens_seen": 2830827520
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.871128229818649e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.3641559630664792,
+      "tokens_seen": 2830958592
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.870325790402825e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.364143948588935,
+      "tokens_seen": 2831089664
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8695233509870005e-05,
+      "loss": 1.2961,
+      "theoretical_loss": 3.3641319348233547,
+      "tokens_seen": 2831220736
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8687209115711765e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.364119921769664,
+      "tokens_seen": 2831351808
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8679184721553525e-05,
+      "loss": 1.3038,
+      "theoretical_loss": 3.364107909427787,
+      "tokens_seen": 2831482880
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8671160327395285e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.3640958977976494,
+      "tokens_seen": 2831613952
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.866313593323704e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.364083886879176,
+      "tokens_seen": 2831745024
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.86551115390788e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.364071876672291,
+      "tokens_seen": 2831876096
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8647087144920558e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.3640598671769197,
+      "tokens_seen": 2832007168
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8639062750762318e-05,
+      "loss": 1.1603,
+      "theoretical_loss": 3.364047858392987,
+      "tokens_seen": 2832138240
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8631038356604077e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.364035850320418,
+      "tokens_seen": 2832269312
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8623013962445837e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.3640238429591376,
+      "tokens_seen": 2832400384
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8614989568287597e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3640118363090705,
+      "tokens_seen": 2832531456
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8606965174129354e-05,
+      "loss": 1.1831,
+      "theoretical_loss": 3.3639998303701417,
+      "tokens_seen": 2832662528
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.859894077997111e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.3639878251422766,
+      "tokens_seen": 2832793600
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.859091638581287e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.3639758206253996,
+      "tokens_seen": 2832924672
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.858289199165463e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.363963816819436,
+      "tokens_seen": 2833055744
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.857486759749639e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.363951813724311,
+      "tokens_seen": 2833186816
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.856684320333815e-05,
+      "loss": 1.2727,
+      "theoretical_loss": 3.3639398113399492,
+      "tokens_seen": 2833317888
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.855881880917991e-05,
+      "loss": 1.1793,
+      "theoretical_loss": 3.363927809666276,
+      "tokens_seen": 2833448960
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.855079441502167e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.363915808703217,
+      "tokens_seen": 2833580032
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8542770020863423e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.3639038084506954,
+      "tokens_seen": 2833711104
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8534745626705183e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.363891808908638,
+      "tokens_seen": 2833842176
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8526721232546943e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3638798100769693,
+      "tokens_seen": 2833973248
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8518696838388703e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.3638678119556147,
+      "tokens_seen": 2834104320
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8510672444230462e-05,
+      "loss": 1.2803,
+      "theoretical_loss": 3.3638558145444986,
+      "tokens_seen": 2834235392
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8502648050072222e-05,
+      "loss": 1.1497,
+      "theoretical_loss": 3.3638438178435464,
+      "tokens_seen": 2834366464
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8494623655913982e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.3638318218526835,
+      "tokens_seen": 2834497536
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8486599261755735e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.363819826571835,
+      "tokens_seen": 2834628608
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8478574867597495e-05,
+      "loss": 1.2944,
+      "theoretical_loss": 3.363807832000926,
+      "tokens_seen": 2834759680
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8470550473439255e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.363795838139881,
+      "tokens_seen": 2834890752
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8462526079281015e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.3637838449886264,
+      "tokens_seen": 2835021824
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8454501685122775e-05,
+      "loss": 1.2882,
+      "theoretical_loss": 3.3637718525470865,
+      "tokens_seen": 2835152896
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8446477290964535e-05,
+      "loss": 1.2949,
+      "theoretical_loss": 3.363759860815187,
+      "tokens_seen": 2835283968
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8438452896806295e-05,
+      "loss": 1.217,
+      "theoretical_loss": 3.3637478697928525,
+      "tokens_seen": 2835415040
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8430428502648048e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.363735879480009,
+      "tokens_seen": 2835546112
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8422404108489808e-05,
+      "loss": 1.2763,
+      "theoretical_loss": 3.3637238898765807,
+      "tokens_seen": 2835677184
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8414379714331568e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.363711900982494,
+      "tokens_seen": 2835808256
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8406355320173328e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.363699912797673,
+      "tokens_seen": 2835939328
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8398330926015088e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3636879253220435,
+      "tokens_seen": 2836070400
+    },
+    {
+      "epoch": 0.72,
+      "objective/train/docs_used": 950760,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1910994052886963,
+      "objective/train/theoretical_loss": 3.3636759385555317,
+      "objective/train/tokens_used": 1206660576,
+      "theoretical_loss": 3.3636759385555317,
+      "tokens_seen": 2836201472
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8390306531856847e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3636759385555317,
+      "tokens_seen": 2836201472
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8382282137698607e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.363663952498061,
+      "tokens_seen": 2836332544
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8374257743540367e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.363651967149558,
+      "tokens_seen": 2836463616
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.836623334938212e-05,
+      "loss": 1.2587,
+      "theoretical_loss": 3.363639982509948,
+      "tokens_seen": 2836594688
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.835820895522388e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.363627998579156,
+      "tokens_seen": 2836725760
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.835018456106564e-05,
+      "loss": 1.2628,
+      "theoretical_loss": 3.363616015357107,
+      "tokens_seen": 2836856832
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.83421601669074e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.363604032843727,
+      "tokens_seen": 2836987904
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.833413577274916e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3635920510389408,
+      "tokens_seen": 2837118976
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.832611137859092e-05,
+      "loss": 1.2514,
+      "theoretical_loss": 3.363580069942674,
+      "tokens_seen": 2837250048
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.831808698443268e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3635680895548523,
+      "tokens_seen": 2837381120
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8310062590274433e-05,
+      "loss": 1.1879,
+      "theoretical_loss": 3.3635561098754003,
+      "tokens_seen": 2837512192
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8302038196116193e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.363544130904244,
+      "tokens_seen": 2837643264
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8294013801957953e-05,
+      "loss": 1.1908,
+      "theoretical_loss": 3.3635321526413087,
+      "tokens_seen": 2837774336
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8285989407799713e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.36352017508652,
+      "tokens_seen": 2837905408
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8277965013641473e-05,
+      "loss": 1.2863,
+      "theoretical_loss": 3.3635081982398027,
+      "tokens_seen": 2838036480
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8269940619483233e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.3634962221010833,
+      "tokens_seen": 2838167552
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8261916225324992e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.363484246670286,
+      "tokens_seen": 2838298624
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8253891831166746e-05,
+      "loss": 1.2967,
+      "theoretical_loss": 3.3634722719473373,
+      "tokens_seen": 2838429696
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8245867437008505e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.363460297932162,
+      "tokens_seen": 2838560768
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8237843042850265e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.363448324624686,
+      "tokens_seen": 2838691840
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8229818648692025e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.3634363520248343,
+      "tokens_seen": 2838822912
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8221794254533785e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.363424380132533,
+      "tokens_seen": 2838953984
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8213769860375545e-05,
+      "loss": 1.1973,
+      "theoretical_loss": 3.3634124089477075,
+      "tokens_seen": 2839085056
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8205745466217305e-05,
+      "loss": 1.1886,
+      "theoretical_loss": 3.363400438470283,
+      "tokens_seen": 2839216128
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8197721072059065e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3633884687001854,
+      "tokens_seen": 2839347200
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8189696677900818e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.36337649963734,
+      "tokens_seen": 2839478272
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8181672283742578e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3633645312816727,
+      "tokens_seen": 2839609344
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8173647889584338e-05,
+      "loss": 1.2102,
+      "theoretical_loss": 3.3633525636331085,
+      "tokens_seen": 2839740416
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8165623495426098e-05,
+      "loss": 1.2335,
+      "theoretical_loss": 3.3633405966915735,
+      "tokens_seen": 2839871488
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8157599101267858e-05,
+      "loss": 1.1941,
+      "theoretical_loss": 3.3633286304569934,
+      "tokens_seen": 2840002560
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8149574707109618e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.363316664929293,
+      "tokens_seen": 2840133632
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8141550312951377e-05,
+      "loss": 1.2805,
+      "theoretical_loss": 3.363304700108399,
+      "tokens_seen": 2840264704
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.813352591879313e-05,
+      "loss": 1.2852,
+      "theoretical_loss": 3.363292735994236,
+      "tokens_seen": 2840395776
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.812550152463489e-05,
+      "loss": 1.2938,
+      "theoretical_loss": 3.3632807725867306,
+      "tokens_seen": 2840526848
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.811747713047665e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.363268809885808,
+      "tokens_seen": 2840657920
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.810945273631841e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3632568478913933,
+      "tokens_seen": 2840788992
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.810142834216017e-05,
+      "loss": 1.2945,
+      "theoretical_loss": 3.3632448866034137,
+      "tokens_seen": 2840920064
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8093403948001927e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.363232926021793,
+      "tokens_seen": 2841051136
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8085379553843687e-05,
+      "loss": 1.3406,
+      "theoretical_loss": 3.3632209661464585,
+      "tokens_seen": 2841182208
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8077355159685447e-05,
+      "loss": 1.1773,
+      "theoretical_loss": 3.363209006977335,
+      "tokens_seen": 2841313280
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8069330765527203e-05,
+      "loss": 1.1747,
+      "theoretical_loss": 3.3631970485143485,
+      "tokens_seen": 2841444352
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8061306371368963e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.3631850907574248,
+      "tokens_seen": 2841575424
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8053281977210723e-05,
+      "loss": 1.2953,
+      "theoretical_loss": 3.3631731337064896,
+      "tokens_seen": 2841706496
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.804525758305248e-05,
+      "loss": 1.312,
+      "theoretical_loss": 3.3631611773614685,
+      "tokens_seen": 2841837568
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.803723318889424e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3631492217222876,
+      "tokens_seen": 2841968640
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8029208794736e-05,
+      "loss": 1.2445,
+      "theoretical_loss": 3.363137266788872,
+      "tokens_seen": 2842099712
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.802118440057776e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.3631253125611487,
+      "tokens_seen": 2842230784
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8013160006419516e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.3631133590390423,
+      "tokens_seen": 2842361856
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.8005135612261276e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.363101406222479,
+      "tokens_seen": 2842492928
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7997111218103032e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.363089454111385,
+      "tokens_seen": 2842624000
+    },
+    {
+      "epoch": 0.72,
+      "objective/train/docs_used": 952946,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3703017234802246,
+      "objective/train/theoretical_loss": 3.3630775027056856,
+      "objective/train/tokens_used": 1213214176,
+      "theoretical_loss": 3.3630775027056856,
+      "tokens_seen": 2842755072
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7989086823944792e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.3630775027056856,
+      "tokens_seen": 2842755072
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7981062429786552e-05,
+      "loss": 1.2712,
+      "theoretical_loss": 3.363065552005307,
+      "tokens_seen": 2842886144
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7973038035628312e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.363053602010175,
+      "tokens_seen": 2843017216
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.796501364147007e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.3630416527202156,
+      "tokens_seen": 2843148288
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7956989247311828e-05,
+      "loss": 1.185,
+      "theoretical_loss": 3.363029704135354,
+      "tokens_seen": 2843279360
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7948964853153585e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.363017756255517,
+      "tokens_seen": 2843410432
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7940940458995345e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.36300580908063,
+      "tokens_seen": 2843541504
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7932916064837105e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.362993862610619,
+      "tokens_seen": 2843672576
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7924891670678864e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.36298191684541,
+      "tokens_seen": 2843803648
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7916867276520624e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3629699717849286,
+      "tokens_seen": 2843934720
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7908842882362384e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3629580274291015,
+      "tokens_seen": 2844065792
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7900818488204144e-05,
+      "loss": 1.3049,
+      "theoretical_loss": 3.362946083777854,
+      "tokens_seen": 2844196864
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7892794094045897e-05,
+      "loss": 1.2554,
+      "theoretical_loss": 3.362934140831112,
+      "tokens_seen": 2844327936
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7884769699887657e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.362922198588802,
+      "tokens_seen": 2844459008
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7876745305729417e-05,
+      "loss": 1.2032,
+      "theoretical_loss": 3.36291025705085,
+      "tokens_seen": 2844590080
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7868720911571177e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3628983162171813,
+      "tokens_seen": 2844721152
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7860696517412937e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.3628863760877223,
+      "tokens_seen": 2844852224
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7852672123254697e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.3628744366623993,
+      "tokens_seen": 2844983296
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7844647729096457e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.362862497941138,
+      "tokens_seen": 2845114368
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.783662333493821e-05,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3628505599238645,
+      "tokens_seen": 2845245440
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.782859894077997e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3628386226105054,
+      "tokens_seen": 2845376512
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.782057454662173e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.362826686000986,
+      "tokens_seen": 2845507584
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.781255015246349e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.3628147500952323,
+      "tokens_seen": 2845638656
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.780452575830525e-05,
+      "loss": 1.2957,
+      "theoretical_loss": 3.3628028148931715,
+      "tokens_seen": 2845769728
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.779650136414701e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3627908803947286,
+      "tokens_seen": 2845900800
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.778847696998877e-05,
+      "loss": 1.1903,
+      "theoretical_loss": 3.36277894659983,
+      "tokens_seen": 2846031872
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 2.7780452575830522e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3627670135084022,
+      "tokens_seen": 2846162944
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7772428181672282e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.362755081120371,
+      "tokens_seen": 2846294016
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7764403787514042e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.362743149435662,
+      "tokens_seen": 2846425088
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7756379393355802e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.362731218454203,
+      "tokens_seen": 2846556160
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7748354999197562e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.3627192881759185,
+      "tokens_seen": 2846687232
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7740330605039322e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.362707358600735,
+      "tokens_seen": 2846818304
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7732306210881082e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.36269542972858,
+      "tokens_seen": 2846949376
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7724281816722842e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.362683501559378,
+      "tokens_seen": 2847080448
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7716257422564595e-05,
+      "loss": 1.1951,
+      "theoretical_loss": 3.3626715740930564,
+      "tokens_seen": 2847211520
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7708233028406355e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3626596473295405,
+      "tokens_seen": 2847342592
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7700208634248115e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3626477212687576,
+      "tokens_seen": 2847473664
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7692184240089875e-05,
+      "loss": 1.2747,
+      "theoretical_loss": 3.3626357959106326,
+      "tokens_seen": 2847604736
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7684159845931635e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.362623871255093,
+      "tokens_seen": 2847735808
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7676135451773394e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.3626119473020646,
+      "tokens_seen": 2847866880
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7668111057615154e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3626000240514737,
+      "tokens_seen": 2847997952
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7660086663456907e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3625881015032464,
+      "tokens_seen": 2848129024
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7652062269298667e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.3625761796573093,
+      "tokens_seen": 2848260096
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7644037875140427e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.3625642585135886,
+      "tokens_seen": 2848391168
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7636013480982187e-05,
+      "loss": 1.242,
+      "theoretical_loss": 3.3625523380720104,
+      "tokens_seen": 2848522240
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7627989086823947e-05,
+      "loss": 1.3058,
+      "theoretical_loss": 3.3625404183325016,
+      "tokens_seen": 2848653312
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7619964692665707e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3625284992949878,
+      "tokens_seen": 2848784384
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7611940298507467e-05,
+      "loss": 1.2134,
+      "theoretical_loss": 3.3625165809593964,
+      "tokens_seen": 2848915456
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7603915904349227e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.3625046633256526,
+      "tokens_seen": 2849046528
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.759589151019098e-05,
+      "loss": 1.1839,
+      "theoretical_loss": 3.3624927463936833,
+      "tokens_seen": 2849177600
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 955085,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3651429414749146,
+      "objective/train/theoretical_loss": 3.3624808301634155,
+      "objective/train/tokens_used": 1219767776,
+      "theoretical_loss": 3.3624808301634155,
+      "tokens_seen": 2849308672
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.758786711603274e-05,
+      "loss": 1.3001,
+      "theoretical_loss": 3.3624808301634155,
+      "tokens_seen": 2849308672
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.75798427218745e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3624689146347744,
+      "tokens_seen": 2849439744
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.757181832771626e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.3624569998076876,
+      "tokens_seen": 2849570816
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.756379393355802e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.3624450856820807,
+      "tokens_seen": 2849701888
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.755576953939978e-05,
+      "loss": 1.1844,
+      "theoretical_loss": 3.36243317225788,
+      "tokens_seen": 2849832960
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.754774514524154e-05,
+      "loss": 1.2401,
+      "theoretical_loss": 3.362421259535013,
+      "tokens_seen": 2849964032
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7539720751083292e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.362409347513405,
+      "tokens_seen": 2850095104
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7531696356925052e-05,
+      "loss": 1.2149,
+      "theoretical_loss": 3.3623974361929836,
+      "tokens_seen": 2850226176
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7523671962766812e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.362385525573674,
+      "tokens_seen": 2850357248
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7515647568608572e-05,
+      "loss": 1.2781,
+      "theoretical_loss": 3.362373615655404,
+      "tokens_seen": 2850488320
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7507623174450332e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.362361706438099,
+      "tokens_seen": 2850619392
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7499598780292092e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.3623497979216865,
+      "tokens_seen": 2850750464
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7491574386133852e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.362337890106092,
+      "tokens_seen": 2850881536
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7483549991975605e-05,
+      "loss": 1.262,
+      "theoretical_loss": 3.362325982991243,
+      "tokens_seen": 2851012608
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7475525597817365e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.362314076577065,
+      "tokens_seen": 2851143680
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7467501203659125e-05,
+      "loss": 1.3057,
+      "theoretical_loss": 3.362302170863486,
+      "tokens_seen": 2851274752
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7459476809500885e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.3622902658504312,
+      "tokens_seen": 2851405824
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7451452415342645e-05,
+      "loss": 1.3128,
+      "theoretical_loss": 3.362278361537828,
+      "tokens_seen": 2851536896
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7443428021184405e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3622664579256023,
+      "tokens_seen": 2851667968
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.743540362702616e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.3622545550136818,
+      "tokens_seen": 2851799040
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.742737923286792e-05,
+      "loss": 1.2012,
+      "theoretical_loss": 3.3622426528019917,
+      "tokens_seen": 2851930112
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7419354838709678e-05,
+      "loss": 1.2748,
+      "theoretical_loss": 3.36223075129046,
+      "tokens_seen": 2852061184
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7411330444551437e-05,
+      "loss": 1.22,
+      "theoretical_loss": 3.3622188504790125,
+      "tokens_seen": 2852192256
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7403306050393197e-05,
+      "loss": 1.165,
+      "theoretical_loss": 3.3622069503675758,
+      "tokens_seen": 2852323328
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7395281656234957e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.362195050956077,
+      "tokens_seen": 2852454400
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7387257262076714e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.3621831522444428,
+      "tokens_seen": 2852585472
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7379232867918474e-05,
+      "loss": 1.1396,
+      "theoretical_loss": 3.3621712542325994,
+      "tokens_seen": 2852716544
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7371208473760234e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.3621593569204737,
+      "tokens_seen": 2852847616
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.736318407960199e-05,
+      "loss": 1.1992,
+      "theoretical_loss": 3.362147460307993,
+      "tokens_seen": 2852978688
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.735515968544375e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.362135564395083,
+      "tokens_seen": 2853109760
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.734713529128551e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.362123669181671,
+      "tokens_seen": 2853240832
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7339110897127266e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.3621117746676834,
+      "tokens_seen": 2853371904
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7331086502969026e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3620998808530476,
+      "tokens_seen": 2853502976
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7323062108810786e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.3620879877376897,
+      "tokens_seen": 2853634048
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7315037714652546e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.362076095321537,
+      "tokens_seen": 2853765120
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7307013320494303e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3620642036045156,
+      "tokens_seen": 2853896192
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7298988926336063e-05,
+      "loss": 1.2182,
+      "theoretical_loss": 3.3620523125865533,
+      "tokens_seen": 2854027264
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.729096453217782e-05,
+      "loss": 1.3028,
+      "theoretical_loss": 3.362040422267576,
+      "tokens_seen": 2854158336
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.728294013801958e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3620285326475106,
+      "tokens_seen": 2854289408
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.727491574386134e-05,
+      "loss": 1.204,
+      "theoretical_loss": 3.3620166437262844,
+      "tokens_seen": 2854420480
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.72668913497031e-05,
+      "loss": 1.1762,
+      "theoretical_loss": 3.362004755503824,
+      "tokens_seen": 2854551552
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.725886695554486e-05,
+      "loss": 1.1403,
+      "theoretical_loss": 3.361992867980056,
+      "tokens_seen": 2854682624
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.725084256138662e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.361980981154908,
+      "tokens_seen": 2854813696
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7242818167228372e-05,
+      "loss": 1.2029,
+      "theoretical_loss": 3.361969095028306,
+      "tokens_seen": 2854944768
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.723479377307013e-05,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3619572096001775,
+      "tokens_seen": 2855075840
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.722676937891189e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.3619453248704487,
+      "tokens_seen": 2855206912
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.721874498475365e-05,
+      "loss": 1.2718,
+      "theoretical_loss": 3.361933440839047,
+      "tokens_seen": 2855337984
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.721072059059541e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3619215575059,
+      "tokens_seen": 2855469056
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.720269619643717e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.361909674870933,
+      "tokens_seen": 2855600128
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.719467180227893e-05,
+      "loss": 1.2583,
+      "theoretical_loss": 3.3618977929340743,
+      "tokens_seen": 2855731200
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 957114,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2522608041763306,
+      "objective/train/theoretical_loss": 3.36188591169525,
+      "objective/train/tokens_used": 1226321376,
+      "theoretical_loss": 3.36188591169525,
+      "tokens_seen": 2855862272
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7186647408120684e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.36188591169525,
+      "tokens_seen": 2855862272
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7178623013962444e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.3618740311543878,
+      "tokens_seen": 2855993344
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7170598619804204e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.361862151311414,
+      "tokens_seen": 2856124416
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7162574225645964e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.361850272166256,
+      "tokens_seen": 2856255488
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7154549831487724e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3618383937188407,
+      "tokens_seen": 2856386560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7146525437329484e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.3618265159690948,
+      "tokens_seen": 2856517632
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7138501043171244e-05,
+      "loss": 1.3163,
+      "theoretical_loss": 3.361814638916946,
+      "tokens_seen": 2856648704
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7130476649013004e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.361802762562321,
+      "tokens_seen": 2856779776
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7122452254854757e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3617908869051463,
+      "tokens_seen": 2856910848
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7114427860696517e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.3617790119453494,
+      "tokens_seen": 2857041920
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7106403466538277e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.3617671376828575,
+      "tokens_seen": 2857172992
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7098379072380037e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3617552641175976,
+      "tokens_seen": 2857304064
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7090354678221796e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.3617433912494965,
+      "tokens_seen": 2857435136
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7082330284063556e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.3617315190784813,
+      "tokens_seen": 2857566208
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7074305889905316e-05,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3617196476044797,
+      "tokens_seen": 2857697280
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.706628149574707e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3617077768274184,
+      "tokens_seen": 2857828352
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.705825710158883e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.361695906747224,
+      "tokens_seen": 2857959424
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.705023270743059e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.3616840373638244,
+      "tokens_seen": 2858090496
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.704220831327235e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.361672168677147,
+      "tokens_seen": 2858221568
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.703418391911411e-05,
+      "loss": 1.187,
+      "theoretical_loss": 3.3616603006871175,
+      "tokens_seen": 2858352640
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.702615952495587e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.3616484333936647,
+      "tokens_seen": 2858483712
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.701813513079763e-05,
+      "loss": 1.163,
+      "theoretical_loss": 3.361636566796715,
+      "tokens_seen": 2858614784
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7010110736639382e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3616247008961957,
+      "tokens_seen": 2858745856
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.7002086342481142e-05,
+      "loss": 1.1672,
+      "theoretical_loss": 3.361612835692034,
+      "tokens_seen": 2858876928
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6994061948322902e-05,
+      "loss": 1.1676,
+      "theoretical_loss": 3.361600971184157,
+      "tokens_seen": 2859008000
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.698603755416466e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3615891073724917,
+      "tokens_seen": 2859139072
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.697801316000642e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.3615772442569662,
+      "tokens_seen": 2859270144
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.696998876584818e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3615653818375066,
+      "tokens_seen": 2859401216
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.696196437168994e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.361553520114041,
+      "tokens_seen": 2859532288
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.69539399775317e-05,
+      "loss": 1.1705,
+      "theoretical_loss": 3.3615416590864964,
+      "tokens_seen": 2859663360
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6945915583373454e-05,
+      "loss": 1.1696,
+      "theoretical_loss": 3.3615297987548,
+      "tokens_seen": 2859794432
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6937891189215214e-05,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3615179391188796,
+      "tokens_seen": 2859925504
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6929866795056974e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.361506080178662,
+      "tokens_seen": 2860056576
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6921842400898734e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.361494221934074,
+      "tokens_seen": 2860187648
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6913818006740494e-05,
+      "loss": 1.2306,
+      "theoretical_loss": 3.361482364385044,
+      "tokens_seen": 2860318720
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6905793612582254e-05,
+      "loss": 1.2202,
+      "theoretical_loss": 3.361470507531499,
+      "tokens_seen": 2860449792
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6897769218424014e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.3614586513733657,
+      "tokens_seen": 2860580864
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6889744824265767e-05,
+      "loss": 1.2814,
+      "theoretical_loss": 3.361446795910572,
+      "tokens_seen": 2860711936
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6881720430107527e-05,
+      "loss": 1.1926,
+      "theoretical_loss": 3.3614349411430453,
+      "tokens_seen": 2860843008
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6873696035949287e-05,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3614230870707127,
+      "tokens_seen": 2860974080
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6865671641791047e-05,
+      "loss": 1.1873,
+      "theoretical_loss": 3.3614112336935023,
+      "tokens_seen": 2861105152
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6857647247632807e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.3613993810113403,
+      "tokens_seen": 2861236224
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6849622853474566e-05,
+      "loss": 1.1986,
+      "theoretical_loss": 3.361387529024155,
+      "tokens_seen": 2861367296
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6841598459316326e-05,
+      "loss": 1.3175,
+      "theoretical_loss": 3.361375677731873,
+      "tokens_seen": 2861498368
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.683357406515808e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.361363827134423,
+      "tokens_seen": 2861629440
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.682554967099984e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.361351977231732,
+      "tokens_seen": 2861760512
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.68175252768416e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.3613401280237265,
+      "tokens_seen": 2861891584
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.680950088268336e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3613282795103347,
+      "tokens_seen": 2862022656
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.680147648852512e-05,
+      "loss": 1.2036,
+      "theoretical_loss": 3.3613164316914843,
+      "tokens_seen": 2862153728
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.679345209436688e-05,
+      "loss": 1.2216,
+      "theoretical_loss": 3.3613045845671023,
+      "tokens_seen": 2862284800
+    },
+    {
+      "epoch": 0.73,
+      "objective/train/docs_used": 959126,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2617950439453125,
+      "objective/train/theoretical_loss": 3.3612927381371165,
+      "objective/train/tokens_used": 1232874976,
+      "theoretical_loss": 3.3612927381371165,
+      "tokens_seen": 2862415872
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.678542770020864e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.3612927381371165,
+      "tokens_seen": 2862415872
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6777403306050395e-05,
+      "loss": 1.2217,
+      "theoretical_loss": 3.361280892401455,
+      "tokens_seen": 2862546944
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.6769378911892152e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.3612690473600435,
+      "tokens_seen": 2862678016
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6761354517733912e-05,
+      "loss": 1.1659,
+      "theoretical_loss": 3.361257203012811,
+      "tokens_seen": 2862809088
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6753330123575672e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.361245359359685,
+      "tokens_seen": 2862940160
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6745305729417432e-05,
+      "loss": 1.326,
+      "theoretical_loss": 3.3612335164005924,
+      "tokens_seen": 2863071232
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.673728133525919e-05,
+      "loss": 1.1906,
+      "theoretical_loss": 3.361221674135461,
+      "tokens_seen": 2863202304
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6729256941100948e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.3612098325642186,
+      "tokens_seen": 2863333376
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6721232546942708e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.361197991686793,
+      "tokens_seen": 2863464448
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6713208152784465e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.361186151503111,
+      "tokens_seen": 2863595520
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6705183758626224e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.361174312013101,
+      "tokens_seen": 2863726592
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6697159364467984e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.36116247321669,
+      "tokens_seen": 2863857664
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6689134970309744e-05,
+      "loss": 1.1598,
+      "theoretical_loss": 3.361150635113806,
+      "tokens_seen": 2863988736
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.66811105761515e-05,
+      "loss": 1.1959,
+      "theoretical_loss": 3.3611387977043763,
+      "tokens_seen": 2864119808
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.667308618199326e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.3611269609883285,
+      "tokens_seen": 2864250880
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.666506178783502e-05,
+      "loss": 1.2092,
+      "theoretical_loss": 3.361115124965591,
+      "tokens_seen": 2864381952
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.665703739367678e-05,
+      "loss": 1.2022,
+      "theoretical_loss": 3.361103289636091,
+      "tokens_seen": 2864513024
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6649012999518537e-05,
+      "loss": 1.2053,
+      "theoretical_loss": 3.361091454999756,
+      "tokens_seen": 2864644096
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6640988605360297e-05,
+      "loss": 1.1473,
+      "theoretical_loss": 3.361079621056514,
+      "tokens_seen": 2864775168
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6632964211202053e-05,
+      "loss": 1.1463,
+      "theoretical_loss": 3.3610677878062925,
+      "tokens_seen": 2864906240
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6624939817043813e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.3610559552490193,
+      "tokens_seen": 2865037312
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6616915422885573e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.361044123384622,
+      "tokens_seen": 2865168384
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6608891028727333e-05,
+      "loss": 1.1351,
+      "theoretical_loss": 3.3610322922130287,
+      "tokens_seen": 2865299456
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6600866634569093e-05,
+      "loss": 1.1485,
+      "theoretical_loss": 3.3610204617341664,
+      "tokens_seen": 2865430528
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.659284224041085e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.3610086319479637,
+      "tokens_seen": 2865561600
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6584817846252606e-05,
+      "loss": 1.1574,
+      "theoretical_loss": 3.3609968028543475,
+      "tokens_seen": 2865692672
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6576793452094366e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3609849744532467,
+      "tokens_seen": 2865823744
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6568769057936126e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.3609731467445885,
+      "tokens_seen": 2865954816
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6560744663777886e-05,
+      "loss": 1.2222,
+      "theoretical_loss": 3.3609613197283,
+      "tokens_seen": 2866085888
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6552720269619646e-05,
+      "loss": 1.1952,
+      "theoretical_loss": 3.36094949340431,
+      "tokens_seen": 2866216960
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6544695875461406e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.360937667772546,
+      "tokens_seen": 2866348032
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.653667148130316e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.360925842832936,
+      "tokens_seen": 2866479104
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.652864708714492e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.360914018585407,
+      "tokens_seen": 2866610176
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.652062269298668e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3609021950298885,
+      "tokens_seen": 2866741248
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.651259829882844e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3608903721663066,
+      "tokens_seen": 2866872320
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.65045739046702e-05,
+      "loss": 1.1813,
+      "theoretical_loss": 3.36087854999459,
+      "tokens_seen": 2867003392
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6496549510511958e-05,
+      "loss": 1.1268,
+      "theoretical_loss": 3.360866728514667,
+      "tokens_seen": 2867134464
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6488525116353718e-05,
+      "loss": 1.1529,
+      "theoretical_loss": 3.3608549077264644,
+      "tokens_seen": 2867265536
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6480500722195478e-05,
+      "loss": 1.2958,
+      "theoretical_loss": 3.3608430876299114,
+      "tokens_seen": 2867396608
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.647247632803723e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.360831268224935,
+      "tokens_seen": 2867527680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.646445193387899e-05,
+      "loss": 1.1954,
+      "theoretical_loss": 3.360819449511463,
+      "tokens_seen": 2867658752
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.645642753972075e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3608076314894237,
+      "tokens_seen": 2867789824
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.644840314556251e-05,
+      "loss": 1.2815,
+      "theoretical_loss": 3.3607958141587457,
+      "tokens_seen": 2867920896
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.644037875140427e-05,
+      "loss": 1.3095,
+      "theoretical_loss": 3.360783997519356,
+      "tokens_seen": 2868051968
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.643235435724603e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.360772181571183,
+      "tokens_seen": 2868183040
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.642432996308779e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.3607603663141545,
+      "tokens_seen": 2868314112
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6416305568929544e-05,
+      "loss": 1.2754,
+      "theoretical_loss": 3.3607485517481983,
+      "tokens_seen": 2868445184
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6408281174771304e-05,
+      "loss": 1.1525,
+      "theoretical_loss": 3.360736737873243,
+      "tokens_seen": 2868576256
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6400256780613064e-05,
+      "loss": 1.2014,
+      "theoretical_loss": 3.360724924689216,
+      "tokens_seen": 2868707328
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6392232386454824e-05,
+      "loss": 1.1072,
+      "theoretical_loss": 3.3607131121960463,
+      "tokens_seen": 2868838400
+    },
+    {
+      "epoch": 0.74,
+      "objective/train/docs_used": 961045,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1844972372055054,
+      "objective/train/theoretical_loss": 3.3607013003936608,
+      "objective/train/tokens_used": 1239428576,
+      "theoretical_loss": 3.3607013003936608,
+      "tokens_seen": 2868969472
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6384207992296583e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3607013003936608,
+      "tokens_seen": 2868969472
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6376183598138343e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.360689489281988,
+      "tokens_seen": 2869100544
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6368159203980103e-05,
+      "loss": 1.1587,
+      "theoretical_loss": 3.360677678860956,
+      "tokens_seen": 2869231616
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6360134809821856e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.360665869130493,
+      "tokens_seen": 2869362688
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6352110415663616e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3606540600905266,
+      "tokens_seen": 2869493760
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6344086021505376e-05,
+      "loss": 1.1909,
+      "theoretical_loss": 3.3606422517409853,
+      "tokens_seen": 2869624832
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6336061627347136e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.360630444081797,
+      "tokens_seen": 2869755904
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6328037233188896e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.36061863711289,
+      "tokens_seen": 2869886976
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6320012839030656e-05,
+      "loss": 1.1254,
+      "theoretical_loss": 3.3606068308341923,
+      "tokens_seen": 2870018048
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6311988444872416e-05,
+      "loss": 1.167,
+      "theoretical_loss": 3.3605950252456323,
+      "tokens_seen": 2870149120
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6303964050714176e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.360583220347138,
+      "tokens_seen": 2870280192
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.629593965655593e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.360571416138637,
+      "tokens_seen": 2870411264
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.628791526239769e-05,
+      "loss": 1.1324,
+      "theoretical_loss": 3.360559612620058,
+      "tokens_seen": 2870542336
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.627989086823945e-05,
+      "loss": 1.1643,
+      "theoretical_loss": 3.3605478097913295,
+      "tokens_seen": 2870673408
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.627186647408121e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.360536007652379,
+      "tokens_seen": 2870804480
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.626384207992297e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.360524206203135,
+      "tokens_seen": 2870935552
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.625581768576473e-05,
+      "loss": 1.1868,
+      "theoretical_loss": 3.360512405443526,
+      "tokens_seen": 2871066624
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6247793291606488e-05,
+      "loss": 1.1855,
+      "theoretical_loss": 3.3605006053734794,
+      "tokens_seen": 2871197696
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.623976889744824e-05,
+      "loss": 1.1723,
+      "theoretical_loss": 3.360488805992924,
+      "tokens_seen": 2871328768
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.623174450329e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.3604770073017884,
+      "tokens_seen": 2871459840
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.622372010913176e-05,
+      "loss": 1.1815,
+      "theoretical_loss": 3.3604652093,
+      "tokens_seen": 2871590912
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.621569571497352e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3604534119874883,
+      "tokens_seen": 2871721984
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.620767132081528e-05,
+      "loss": 1.1484,
+      "theoretical_loss": 3.36044161536418,
+      "tokens_seen": 2871853056
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.619964692665704e-05,
+      "loss": 1.145,
+      "theoretical_loss": 3.360429819430004,
+      "tokens_seen": 2871984128
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.61916225324988e-05,
+      "loss": 1.205,
+      "theoretical_loss": 3.3604180241848893,
+      "tokens_seen": 2872115200
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.618359813834056e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.3604062296287633,
+      "tokens_seen": 2872246272
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6175573744182314e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.360394435761555,
+      "tokens_seen": 2872377344
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6167549350024074e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3603826425831924,
+      "tokens_seen": 2872508416
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6159524955865834e-05,
+      "loss": 1.201,
+      "theoretical_loss": 3.360370850093604,
+      "tokens_seen": 2872639488
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6151500561707594e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.3603590582927176,
+      "tokens_seen": 2872770560
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6143476167549354e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.3603472671804617,
+      "tokens_seen": 2872901632
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6135451773391113e-05,
+      "loss": 1.1733,
+      "theoretical_loss": 3.3603354767567657,
+      "tokens_seen": 2873032704
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6127427379232873e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.3603236870215567,
+      "tokens_seen": 2873163776
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6119402985074626e-05,
+      "loss": 1.2066,
+      "theoretical_loss": 3.3603118979747637,
+      "tokens_seen": 2873294848
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6111378590916386e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3603001096163148,
+      "tokens_seen": 2873425920
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6103354196758146e-05,
+      "loss": 1.1451,
+      "theoretical_loss": 3.360288321946139,
+      "tokens_seen": 2873556992
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6095329802599906e-05,
+      "loss": 1.1506,
+      "theoretical_loss": 3.360276534964164,
+      "tokens_seen": 2873688064
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6087305408441666e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3602647486703185,
+      "tokens_seen": 2873819136
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6079281014283426e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3602529630645313,
+      "tokens_seen": 2873950208
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6071256620125182e-05,
+      "loss": 1.1994,
+      "theoretical_loss": 3.3602411781467305,
+      "tokens_seen": 2874081280
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.606323222596694e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3602293939168444,
+      "tokens_seen": 2874212352
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.60552078318087e-05,
+      "loss": 1.1654,
+      "theoretical_loss": 3.360217610374802,
+      "tokens_seen": 2874343424
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.604718343765046e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.360205827520531,
+      "tokens_seen": 2874474496
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.603915904349222e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.3601940453539605,
+      "tokens_seen": 2874605568
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6031134649333975e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3601822638750196,
+      "tokens_seen": 2874736640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6023110255175735e-05,
+      "loss": 1.1558,
+      "theoretical_loss": 3.3601704830836354,
+      "tokens_seen": 2874867712
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6015085861017495e-05,
+      "loss": 1.1648,
+      "theoretical_loss": 3.3601587029797373,
+      "tokens_seen": 2874998784
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.6007061466859255e-05,
+      "loss": 1.1357,
+      "theoretical_loss": 3.3601469235632537,
+      "tokens_seen": 2875129856
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.599903707270101e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.3601351448341132,
+      "tokens_seen": 2875260928
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.599101267854277e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.3601233667922443,
+      "tokens_seen": 2875392000
+    },
+    {
+      "epoch": 0.74,
+      "objective/train/docs_used": 963256,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3264129161834717,
+      "objective/train/theoretical_loss": 3.360111589437576,
+      "objective/train/tokens_used": 1245982176,
+      "theoretical_loss": 3.360111589437576,
+      "tokens_seen": 2875523072
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5982988284384528e-05,
+      "loss": 1.2682,
+      "theoretical_loss": 3.360111589437576,
+      "tokens_seen": 2875523072
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5974963890226288e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3600998127700357,
+      "tokens_seen": 2875654144
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5966939496068048e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.360088036789553,
+      "tokens_seen": 2875785216
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5958915101909808e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.360076261496056,
+      "tokens_seen": 2875916288
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5950890707751568e-05,
+      "loss": 1.176,
+      "theoretical_loss": 3.3600644868894745,
+      "tokens_seen": 2876047360
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5942866313593324e-05,
+      "loss": 1.1746,
+      "theoretical_loss": 3.3600527129697353,
+      "tokens_seen": 2876178432
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.593484191943508e-05,
+      "loss": 1.1739,
+      "theoretical_loss": 3.360040939736768,
+      "tokens_seen": 2876309504
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.592681752527684e-05,
+      "loss": 1.1326,
+      "theoretical_loss": 3.360029167190502,
+      "tokens_seen": 2876440576
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.59187931311186e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.360017395330864,
+      "tokens_seen": 2876571648
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.591076873696036e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.3600056241577847,
+      "tokens_seen": 2876702720
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.590274434280212e-05,
+      "loss": 1.1705,
+      "theoretical_loss": 3.3599938536711917,
+      "tokens_seen": 2876833792
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.589471994864388e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.3599820838710137,
+      "tokens_seen": 2876964864
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5886695554485633e-05,
+      "loss": 1.278,
+      "theoretical_loss": 3.3599703147571796,
+      "tokens_seen": 2877095936
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5878671160327393e-05,
+      "loss": 1.1946,
+      "theoretical_loss": 3.3599585463296178,
+      "tokens_seen": 2877227008
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5870646766169153e-05,
+      "loss": 1.1495,
+      "theoretical_loss": 3.3599467785882577,
+      "tokens_seen": 2877358080
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5862622372010913e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.3599350115330275,
+      "tokens_seen": 2877489152
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5854597977852673e-05,
+      "loss": 1.1616,
+      "theoretical_loss": 3.359923245163856,
+      "tokens_seen": 2877620224
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5846573583694433e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3599114794806724,
+      "tokens_seen": 2877751296
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5838549189536193e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.359899714483405,
+      "tokens_seen": 2877882368
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5830524795377953e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.3598879501719825,
+      "tokens_seen": 2878013440
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5822500401219706e-05,
+      "loss": 1.3088,
+      "theoretical_loss": 3.359876186546334,
+      "tokens_seen": 2878144512
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5814476007061466e-05,
+      "loss": 1.2074,
+      "theoretical_loss": 3.359864423606388,
+      "tokens_seen": 2878275584
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5806451612903226e-05,
+      "loss": 1.2981,
+      "theoretical_loss": 3.3598526613520736,
+      "tokens_seen": 2878406656
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5798427218744985e-05,
+      "loss": 1.1576,
+      "theoretical_loss": 3.3598408997833196,
+      "tokens_seen": 2878537728
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5790402824586745e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.3598291389000545,
+      "tokens_seen": 2878668800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5782378430428505e-05,
+      "loss": 1.1547,
+      "theoretical_loss": 3.3598173787022074,
+      "tokens_seen": 2878799872
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5774354036270265e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.359805619189707,
+      "tokens_seen": 2878930944
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5766329642112018e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3597938603624824,
+      "tokens_seen": 2879062016
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 2.5758305247953778e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3597821022204624,
+      "tokens_seen": 2879193088
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5750280853795538e-05,
+      "loss": 1.2806,
+      "theoretical_loss": 3.359770344763576,
+      "tokens_seen": 2879324160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5742256459637298e-05,
+      "loss": 1.3355,
+      "theoretical_loss": 3.3597585879917515,
+      "tokens_seen": 2879455232
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5734232065479058e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.3597468319049186,
+      "tokens_seen": 2879586304
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5726207671320818e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.3597350765030054,
+      "tokens_seen": 2879717376
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5718183277162578e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.3597233217859417,
+      "tokens_seen": 2879848448
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5710158883004338e-05,
+      "loss": 1.2365,
+      "theoretical_loss": 3.3597115677536555,
+      "tokens_seen": 2879979520
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.570213448884609e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.359699814406077,
+      "tokens_seen": 2880110592
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.569411009468785e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.3596880617431335,
+      "tokens_seen": 2880241664
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.568608570052961e-05,
+      "loss": 1.2726,
+      "theoretical_loss": 3.3596763097647555,
+      "tokens_seen": 2880372736
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.567806130637137e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.359664558470871,
+      "tokens_seen": 2880503808
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.567003691221313e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3596528078614094,
+      "tokens_seen": 2880634880
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.566201251805489e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3596410579363,
+      "tokens_seen": 2880765952
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.565398812389665e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.359629308695471,
+      "tokens_seen": 2880897024
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5645963729738403e-05,
+      "loss": 1.2362,
+      "theoretical_loss": 3.359617560138852,
+      "tokens_seen": 2881028096
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5637939335580163e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.3596058122663717,
+      "tokens_seen": 2881159168
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5629914941421923e-05,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3595940650779594,
+      "tokens_seen": 2881290240
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5621890547263683e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.359582318573544,
+      "tokens_seen": 2881421312
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5613866153105443e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.359570572753055,
+      "tokens_seen": 2881552384
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5605841758947203e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3595588276164206,
+      "tokens_seen": 2881683456
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5597817364788963e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.3595470831635708,
+      "tokens_seen": 2881814528
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5589792970630716e-05,
+      "loss": 1.2798,
+      "theoretical_loss": 3.359535339394434,
+      "tokens_seen": 2881945600
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 965415,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1915462017059326,
+      "objective/train/theoretical_loss": 3.3595235963089394,
+      "objective/train/tokens_used": 1252535776,
+      "theoretical_loss": 3.3595235963089394,
+      "tokens_seen": 2882076672
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5581768576472476e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.3595235963089394,
+      "tokens_seen": 2882076672
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5573744182314236e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.359511853907017,
+      "tokens_seen": 2882207744
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5565719788155996e-05,
+      "loss": 1.2994,
+      "theoretical_loss": 3.3595001121885946,
+      "tokens_seen": 2882338816
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5557695393997755e-05,
+      "loss": 1.2891,
+      "theoretical_loss": 3.3594883711536023,
+      "tokens_seen": 2882469888
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5549670999839515e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.359476630801969,
+      "tokens_seen": 2882600960
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5541646605681275e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.3594648911336233,
+      "tokens_seen": 2882732032
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5533622211523035e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.359453152148495,
+      "tokens_seen": 2882863104
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.552559781736479e-05,
+      "loss": 1.1609,
+      "theoretical_loss": 3.359441413846513,
+      "tokens_seen": 2882994176
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5517573423206548e-05,
+      "loss": 1.175,
+      "theoretical_loss": 3.3594296762276072,
+      "tokens_seen": 2883125248
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5509549029048308e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.359417939291706,
+      "tokens_seen": 2883256320
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5501524634890068e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.359406203038738,
+      "tokens_seen": 2883387392
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5493500240731828e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.359394467468634,
+      "tokens_seen": 2883518464
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5485475846573588e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3593827325813224,
+      "tokens_seen": 2883649536
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5477451452415348e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3593709983767326,
+      "tokens_seen": 2883780608
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.54694270582571e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.359359264854793,
+      "tokens_seen": 2883911680
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.546140266409886e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.359347532015435,
+      "tokens_seen": 2884042752
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.545337826994062e-05,
+      "loss": 1.325,
+      "theoretical_loss": 3.3593357998585853,
+      "tokens_seen": 2884173824
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.544535387578238e-05,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3593240683841747,
+      "tokens_seen": 2884304896
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.543732948162414e-05,
+      "loss": 1.1533,
+      "theoretical_loss": 3.359312337592132,
+      "tokens_seen": 2884435968
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.54293050874659e-05,
+      "loss": 1.1934,
+      "theoretical_loss": 3.3593006074823872,
+      "tokens_seen": 2884567040
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5421280693307657e-05,
+      "loss": 1.1866,
+      "theoretical_loss": 3.3592888780548686,
+      "tokens_seen": 2884698112
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5413256299149413e-05,
+      "loss": 1.3163,
+      "theoretical_loss": 3.359277149309506,
+      "tokens_seen": 2884829184
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5405231904991173e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.359265421246229,
+      "tokens_seen": 2884960256
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5397207510832933e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.359253693864967,
+      "tokens_seen": 2885091328
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5389183116674693e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.3592419671656484,
+      "tokens_seen": 2885222400
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5381158722516453e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.359230241148204,
+      "tokens_seen": 2885353472
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.537313432835821e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.359218515812562,
+      "tokens_seen": 2885484544
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.536510993419997e-05,
+      "loss": 1.186,
+      "theoretical_loss": 3.359206791158652,
+      "tokens_seen": 2885615616
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.535708554004173e-05,
+      "loss": 1.1723,
+      "theoretical_loss": 3.3591950671864033,
+      "tokens_seen": 2885746688
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5349061145883486e-05,
+      "loss": 1.2901,
+      "theoretical_loss": 3.3591833438957464,
+      "tokens_seen": 2885877760
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5341036751725246e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.359171621286609,
+      "tokens_seen": 2886008832
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5333012357567006e-05,
+      "loss": 1.3276,
+      "theoretical_loss": 3.359159899358922,
+      "tokens_seen": 2886139904
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5324987963408762e-05,
+      "loss": 1.141,
+      "theoretical_loss": 3.3591481781126147,
+      "tokens_seen": 2886270976
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5316963569250522e-05,
+      "loss": 1.2428,
+      "theoretical_loss": 3.3591364575476153,
+      "tokens_seen": 2886402048
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5308939175092282e-05,
+      "loss": 1.2655,
+      "theoretical_loss": 3.359124737663855,
+      "tokens_seen": 2886533120
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5300914780934042e-05,
+      "loss": 1.2775,
+      "theoretical_loss": 3.3591130184612616,
+      "tokens_seen": 2886664192
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.52928903867758e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.359101299939766,
+      "tokens_seen": 2886795264
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.528486599261756e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.3590895820992963,
+      "tokens_seen": 2886926336
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5276841598459315e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.359077864939783,
+      "tokens_seen": 2887057408
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5268817204301075e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.3590661484611557,
+      "tokens_seen": 2887188480
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5260792810142835e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.359054432663344,
+      "tokens_seen": 2887319552
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5252768415984595e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.3590427175462763,
+      "tokens_seen": 2887450624
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5244744021826355e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.359031003109883,
+      "tokens_seen": 2887581696
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5236719627668114e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3590192893540936,
+      "tokens_seen": 2887712768
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5228695233509868e-05,
+      "loss": 1.17,
+      "theoretical_loss": 3.3590075762788376,
+      "tokens_seen": 2887843840
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5220670839351628e-05,
+      "loss": 1.202,
+      "theoretical_loss": 3.3589958638840445,
+      "tokens_seen": 2887974912
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5212646445193387e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3589841521696444,
+      "tokens_seen": 2888105984
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5204622051035147e-05,
+      "loss": 1.3092,
+      "theoretical_loss": 3.3589724411355664,
+      "tokens_seen": 2888237056
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5196597656876907e-05,
+      "loss": 1.2397,
+      "theoretical_loss": 3.3589607307817397,
+      "tokens_seen": 2888368128
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5188573262718667e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.358949021108095,
+      "tokens_seen": 2888499200
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 967898,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.290891170501709,
+      "objective/train/theoretical_loss": 3.358937312114561,
+      "objective/train/tokens_used": 1259089376,
+      "theoretical_loss": 3.358937312114561,
+      "tokens_seen": 2888630272
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5180548868560427e-05,
+      "loss": 1.2758,
+      "theoretical_loss": 3.358937312114561,
+      "tokens_seen": 2888630272
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.517252447440218e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3589256038010675,
+      "tokens_seen": 2888761344
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.516450008024394e-05,
+      "loss": 1.274,
+      "theoretical_loss": 3.3589138961675444,
+      "tokens_seen": 2888892416
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.51564756860857e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.3589021892139215,
+      "tokens_seen": 2889023488
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.514845129192746e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3588904829401285,
+      "tokens_seen": 2889154560
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.514042689776922e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3588787773460944,
+      "tokens_seen": 2889285632
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.513240250361098e-05,
+      "loss": 1.1908,
+      "theoretical_loss": 3.3588670724317495,
+      "tokens_seen": 2889416704
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.512437810945274e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3588553681970232,
+      "tokens_seen": 2889547776
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5116353715294493e-05,
+      "loss": 1.29,
+      "theoretical_loss": 3.358843664641846,
+      "tokens_seen": 2889678848
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5108329321136253e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.358831961766146,
+      "tokens_seen": 2889809920
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5100304926978013e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.3588202595698546,
+      "tokens_seen": 2889940992
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5092280532819772e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.3588085580529006,
+      "tokens_seen": 2890072064
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5084256138661532e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.3587968572152143,
+      "tokens_seen": 2890203136
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5076231744503292e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.358785157056725,
+      "tokens_seen": 2890334208
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5068207350345052e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3587734575773625,
+      "tokens_seen": 2890465280
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5060182956186812e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.3587617587770566,
+      "tokens_seen": 2890596352
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5052158562028565e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.358750060655738,
+      "tokens_seen": 2890727424
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5044134167870325e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.358738363213335,
+      "tokens_seen": 2890858496
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5036109773712085e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.358726666449779,
+      "tokens_seen": 2890989568
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5028085379553845e-05,
+      "loss": 1.3331,
+      "theoretical_loss": 3.358714970364998,
+      "tokens_seen": 2891120640
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5020060985395605e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.3587032749589234,
+      "tokens_seen": 2891251712
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5012036591237365e-05,
+      "loss": 1.1624,
+      "theoretical_loss": 3.3586915802314845,
+      "tokens_seen": 2891382784
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.5004012197079125e-05,
+      "loss": 1.146,
+      "theoretical_loss": 3.358679886182611,
+      "tokens_seen": 2891513856
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.499598780292088e-05,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3586681928122326,
+      "tokens_seen": 2891644928
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.498796340876264e-05,
+      "loss": 1.265,
+      "theoretical_loss": 3.35865650012028,
+      "tokens_seen": 2891776000
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4979939014604398e-05,
+      "loss": 1.1713,
+      "theoretical_loss": 3.358644808106683,
+      "tokens_seen": 2891907072
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4971914620446157e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.3586331167713706,
+      "tokens_seen": 2892038144
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4963890226287917e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.358621426114273,
+      "tokens_seen": 2892169216
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4955865832129677e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.35860973613532,
+      "tokens_seen": 2892300288
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4947841437971434e-05,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3585980468344427,
+      "tokens_seen": 2892431360
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4939817043813194e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.35858635821157,
+      "tokens_seen": 2892562432
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4931792649654954e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.358574670266632,
+      "tokens_seen": 2892693504
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.492376825549671e-05,
+      "loss": 1.2662,
+      "theoretical_loss": 3.3585629829995587,
+      "tokens_seen": 2892824576
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.491574386133847e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.35855129641028,
+      "tokens_seen": 2892955648
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.490771946718023e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.358539610498726,
+      "tokens_seen": 2893086720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.489969507302199e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.358527925264827,
+      "tokens_seen": 2893217792
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4891670678863746e-05,
+      "loss": 1.2597,
+      "theoretical_loss": 3.3585162407085125,
+      "tokens_seen": 2893348864
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4883646284705506e-05,
+      "loss": 1.1563,
+      "theoretical_loss": 3.358504556829713,
+      "tokens_seen": 2893479936
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4875621890547266e-05,
+      "loss": 1.1591,
+      "theoretical_loss": 3.3584928736283577,
+      "tokens_seen": 2893611008
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4867597496389026e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.3584811911043775,
+      "tokens_seen": 2893742080
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4859573102230783e-05,
+      "loss": 1.1958,
+      "theoretical_loss": 3.358469509257702,
+      "tokens_seen": 2893873152
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4851548708072543e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.3584578280882615,
+      "tokens_seen": 2894004224
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4843524313914302e-05,
+      "loss": 1.1519,
+      "theoretical_loss": 3.358446147595986,
+      "tokens_seen": 2894135296
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.483549991975606e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.358434467780805,
+      "tokens_seen": 2894266368
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.482747552559782e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.35842278864265,
+      "tokens_seen": 2894397440
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.481945113143958e-05,
+      "loss": 1.1545,
+      "theoretical_loss": 3.3584111101814496,
+      "tokens_seen": 2894528512
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.481142673728134e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.3583994323971345,
+      "tokens_seen": 2894659584
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4803402343123095e-05,
+      "loss": 1.203,
+      "theoretical_loss": 3.358387755289635,
+      "tokens_seen": 2894790656
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4795377948964855e-05,
+      "loss": 1.3403,
+      "theoretical_loss": 3.358376078858881,
+      "tokens_seen": 2894921728
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4787353554806615e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3583644031048028,
+      "tokens_seen": 2895052800
+    },
+    {
+      "epoch": 0.75,
+      "objective/train/docs_used": 970103,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2583250999450684,
+      "objective/train/theoretical_loss": 3.358352728027331,
+      "objective/train/tokens_used": 1265642976,
+      "theoretical_loss": 3.358352728027331,
+      "tokens_seen": 2895183872
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.4779329160648375e-05,
+      "loss": 1.294,
+      "theoretical_loss": 3.358352728027331,
+      "tokens_seen": 2895183872
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.477130476649013e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.3583410536263942,
+      "tokens_seen": 2895314944
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.476328037233189e-05,
+      "loss": 1.1726,
+      "theoretical_loss": 3.3583293799019245,
+      "tokens_seen": 2895446016
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 2.475525597817365e-05,
+      "loss": 1.1835,
+      "theoretical_loss": 3.3583177068538506,
+      "tokens_seen": 2895577088
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4747231584015408e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.3583060344821036,
+      "tokens_seen": 2895708160
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4739207189857168e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.358294362786613,
+      "tokens_seen": 2895839232
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4731182795698928e-05,
+      "loss": 1.1788,
+      "theoretical_loss": 3.35828269176731,
+      "tokens_seen": 2895970304
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4723158401540687e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.358271021424124,
+      "tokens_seen": 2896101376
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4715134007382444e-05,
+      "loss": 1.1996,
+      "theoretical_loss": 3.358259351756985,
+      "tokens_seen": 2896232448
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4707109613224204e-05,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3582476827658243,
+      "tokens_seen": 2896363520
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4699085219065964e-05,
+      "loss": 1.1874,
+      "theoretical_loss": 3.358236014450571,
+      "tokens_seen": 2896494592
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.469106082490772e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.3582243468111566,
+      "tokens_seen": 2896625664
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.468303643074948e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3582126798475103,
+      "tokens_seen": 2896756736
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.467501203659124e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.358201013559563,
+      "tokens_seen": 2896887808
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4666987642432997e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3581893479472447,
+      "tokens_seen": 2897018880
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4658963248274757e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.3581776830104855,
+      "tokens_seen": 2897149952
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4650938854116516e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.358166018749216,
+      "tokens_seen": 2897281024
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4642914459958273e-05,
+      "loss": 1.2283,
+      "theoretical_loss": 3.358154355163367,
+      "tokens_seen": 2897412096
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4634890065800033e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.358142692252868,
+      "tokens_seen": 2897543168
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4626865671641793e-05,
+      "loss": 1.2178,
+      "theoretical_loss": 3.35813103001765,
+      "tokens_seen": 2897674240
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.461884127748355e-05,
+      "loss": 1.2526,
+      "theoretical_loss": 3.358119368457643,
+      "tokens_seen": 2897805312
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.461081688332531e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.358107707572777,
+      "tokens_seen": 2897936384
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.460279248916707e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.3580960473629835,
+      "tokens_seen": 2898067456
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4594768095008826e-05,
+      "loss": 1.2158,
+      "theoretical_loss": 3.3580843878281916,
+      "tokens_seen": 2898198528
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4586743700850586e-05,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3580727289683328,
+      "tokens_seen": 2898329600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4578719306692345e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.3580610707833367,
+      "tokens_seen": 2898460672
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4570694912534102e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.358049413273134,
+      "tokens_seen": 2898591744
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4562670518375862e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.358037756437656,
+      "tokens_seen": 2898722816
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4554646124217622e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.3580261002768315,
+      "tokens_seen": 2898853888
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.454662173005938e-05,
+      "loss": 1.2837,
+      "theoretical_loss": 3.3580144447905917,
+      "tokens_seen": 2898984960
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4538597335901138e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.358002789978868,
+      "tokens_seen": 2899116032
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4530572941742898e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.3579911358415893,
+      "tokens_seen": 2899247104
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4522548547584658e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3579794823786866,
+      "tokens_seen": 2899378176
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4514524153426418e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.357967829590091,
+      "tokens_seen": 2899509248
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4506499759268174e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.3579561774757325,
+      "tokens_seen": 2899640320
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4498475365109934e-05,
+      "loss": 1.169,
+      "theoretical_loss": 3.357944526035542,
+      "tokens_seen": 2899771392
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4490450970951694e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3579328752694493,
+      "tokens_seen": 2899902464
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4482426576793454e-05,
+      "loss": 1.3183,
+      "theoretical_loss": 3.3579212251773853,
+      "tokens_seen": 2900033536
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.447440218263521e-05,
+      "loss": 1.1979,
+      "theoretical_loss": 3.357909575759281,
+      "tokens_seen": 2900164608
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.446637778847697e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.357897927015066,
+      "tokens_seen": 2900295680
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.445835339431873e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.357886278944672,
+      "tokens_seen": 2900426752
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4450329000160487e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.357874631548029,
+      "tokens_seen": 2900557824
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4442304606002247e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.3578629848250667,
+      "tokens_seen": 2900688896
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4434280211844007e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.3578513387757174,
+      "tokens_seen": 2900819968
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4426255817685767e-05,
+      "loss": 1.1853,
+      "theoretical_loss": 3.3578396933999106,
+      "tokens_seen": 2900951040
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4418231423527523e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.3578280486975776,
+      "tokens_seen": 2901082112
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4410207029369283e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.357816404668648,
+      "tokens_seen": 2901213184
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4402182635211043e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3578047613130533,
+      "tokens_seen": 2901344256
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4394158241052803e-05,
+      "loss": 1.2037,
+      "theoretical_loss": 3.357793118630724,
+      "tokens_seen": 2901475328
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.438613384689456e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.3577814766215908,
+      "tokens_seen": 2901606400
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/docs_used": 972373,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2524323463439941,
+      "objective/train/theoretical_loss": 3.357769835285584,
+      "objective/train/tokens_used": 1272196576,
+      "theoretical_loss": 3.357769835285584,
+      "tokens_seen": 2901737472
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.437810945273632e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.357769835285584,
+      "tokens_seen": 2901737472
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.437008505857808e-05,
+      "loss": 1.2674,
+      "theoretical_loss": 3.357758194622635,
+      "tokens_seen": 2901868544
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4362060664419836e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.3577465546326732,
+      "tokens_seen": 2901999616
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4354036270261596e-05,
+      "loss": 1.1523,
+      "theoretical_loss": 3.3577349153156306,
+      "tokens_seen": 2902130688
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4346011876103356e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3577232766714373,
+      "tokens_seen": 2902261760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4337987481945116e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.357711638700024,
+      "tokens_seen": 2902392832
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4329963087786872e-05,
+      "loss": 1.2018,
+      "theoretical_loss": 3.357700001401322,
+      "tokens_seen": 2902523904
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4321938693628632e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.357688364775261,
+      "tokens_seen": 2902654976
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4313914299470392e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.357676728821773,
+      "tokens_seen": 2902786048
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4305889905312152e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.3576650935407875,
+      "tokens_seen": 2902917120
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4297865511153908e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.357653458932236,
+      "tokens_seen": 2903048192
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4289841116995668e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3576418249960494,
+      "tokens_seen": 2903179264
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4281816722837428e-05,
+      "loss": 1.1846,
+      "theoretical_loss": 3.357630191732158,
+      "tokens_seen": 2903310336
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4273792328679185e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.357618559140493,
+      "tokens_seen": 2903441408
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4265767934520945e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.357606927220985,
+      "tokens_seen": 2903572480
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4257743540362704e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.357595295973565,
+      "tokens_seen": 2903703552
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4249719146204464e-05,
+      "loss": 1.2366,
+      "theoretical_loss": 3.3575836653981637,
+      "tokens_seen": 2903834624
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.424169475204622e-05,
+      "loss": 1.3314,
+      "theoretical_loss": 3.357572035494712,
+      "tokens_seen": 2903965696
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.423367035788798e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.357560406263141,
+      "tokens_seen": 2904096768
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.422564596372974e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3575487777033803,
+      "tokens_seen": 2904227840
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.42176215695715e-05,
+      "loss": 1.1802,
+      "theoretical_loss": 3.3575371498153626,
+      "tokens_seen": 2904358912
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4209597175413257e-05,
+      "loss": 1.19,
+      "theoretical_loss": 3.3575255225990173,
+      "tokens_seen": 2904489984
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4201572781255017e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.3575138960542765,
+      "tokens_seen": 2904621056
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4193548387096777e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.35750227018107,
+      "tokens_seen": 2904752128
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4185523992938533e-05,
+      "loss": 1.2922,
+      "theoretical_loss": 3.35749064497933,
+      "tokens_seen": 2904883200
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4177499598780293e-05,
+      "loss": 1.1867,
+      "theoretical_loss": 3.357479020448986,
+      "tokens_seen": 2905014272
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4169475204622053e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.35746739658997,
+      "tokens_seen": 2905145344
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4161450810463813e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.3574557734022124,
+      "tokens_seen": 2905276416
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.415342641630557e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.357444150885644,
+      "tokens_seen": 2905407488
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.414540202214733e-05,
+      "loss": 1.289,
+      "theoretical_loss": 3.3574325290401963,
+      "tokens_seen": 2905538560
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.413737762798909e-05,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3574209078658,
+      "tokens_seen": 2905669632
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.412935323383085e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.357409287362386,
+      "tokens_seen": 2905800704
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4121328839672606e-05,
+      "loss": 1.1944,
+      "theoretical_loss": 3.3573976675298858,
+      "tokens_seen": 2905931776
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4113304445514366e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.35738604836823,
+      "tokens_seen": 2906062848
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4105280051356126e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3573744298773494,
+      "tokens_seen": 2906193920
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4097255657197882e-05,
+      "loss": 1.2398,
+      "theoretical_loss": 3.357362812057175,
+      "tokens_seen": 2906324992
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4089231263039642e-05,
+      "loss": 1.1398,
+      "theoretical_loss": 3.3573511949076384,
+      "tokens_seen": 2906456064
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4081206868881402e-05,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3573395784286704,
+      "tokens_seen": 2906587136
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4073182474723162e-05,
+      "loss": 1.2575,
+      "theoretical_loss": 3.357327962620202,
+      "tokens_seen": 2906718208
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.406515808056492e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.357316347482164,
+      "tokens_seen": 2906849280
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.405713368640668e-05,
+      "loss": 1.2549,
+      "theoretical_loss": 3.3573047330144883,
+      "tokens_seen": 2906980352
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4049109292248438e-05,
+      "loss": 1.2633,
+      "theoretical_loss": 3.357293119217105,
+      "tokens_seen": 2907111424
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4041084898090198e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.3572815060899455,
+      "tokens_seen": 2907242496
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4033060503931955e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3572698936329415,
+      "tokens_seen": 2907373568
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4025036109773715e-05,
+      "loss": 1.1793,
+      "theoretical_loss": 3.3572582818460233,
+      "tokens_seen": 2907504640
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.4017011715615474e-05,
+      "loss": 1.2079,
+      "theoretical_loss": 3.357246670729123,
+      "tokens_seen": 2907635712
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.400898732145723e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.35723506028217,
+      "tokens_seen": 2907766784
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.400096292729899e-05,
+      "loss": 1.186,
+      "theoretical_loss": 3.3572234505050975,
+      "tokens_seen": 2907897856
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.399293853314075e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3572118413978354,
+      "tokens_seen": 2908028928
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3984914138982507e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.3572002329603152,
+      "tokens_seen": 2908160000
+    },
+    {
+      "epoch": 0.76,
+      "objective/train/docs_used": 974455,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2643496990203857,
+      "objective/train/theoretical_loss": 3.3571886251924683,
+      "objective/train/tokens_used": 1278750176,
+      "theoretical_loss": 3.3571886251924683,
+      "tokens_seen": 2908291072
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3976889744824267e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3571886251924683,
+      "tokens_seen": 2908291072
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3968865350666027e-05,
+      "loss": 1.144,
+      "theoretical_loss": 3.3571770180942253,
+      "tokens_seen": 2908422144
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3960840956507784e-05,
+      "loss": 1.2676,
+      "theoretical_loss": 3.357165411665518,
+      "tokens_seen": 2908553216
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3952816562349544e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.357153805906277,
+      "tokens_seen": 2908684288
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.39447921681913e-05,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3571422008164347,
+      "tokens_seen": 2908815360
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.393676777403306e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.357130596395921,
+      "tokens_seen": 2908946432
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.392874337987482e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.3571189926446676,
+      "tokens_seen": 2909077504
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.392071898571658e-05,
+      "loss": 1.2503,
+      "theoretical_loss": 3.3571073895626062,
+      "tokens_seen": 2909208576
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3912694591558336e-05,
+      "loss": 1.1793,
+      "theoretical_loss": 3.3570957871496674,
+      "tokens_seen": 2909339648
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3904670197400096e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.3570841854057827,
+      "tokens_seen": 2909470720
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3896645803241856e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.3570725843308833,
+      "tokens_seen": 2909601792
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3888621409083613e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.357060983924901,
+      "tokens_seen": 2909732864
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3880597014925373e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3570493841877664,
+      "tokens_seen": 2909863936
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3872572620767132e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3570377851194113,
+      "tokens_seen": 2909995008
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3864548226608892e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.357026186719767,
+      "tokens_seen": 2910126080
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.385652383245065e-05,
+      "loss": 1.2911,
+      "theoretical_loss": 3.3570145889887644,
+      "tokens_seen": 2910257152
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.384849943829241e-05,
+      "loss": 1.1602,
+      "theoretical_loss": 3.3570029919263353,
+      "tokens_seen": 2910388224
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.384047504413417e-05,
+      "loss": 1.1718,
+      "theoretical_loss": 3.3569913955324107,
+      "tokens_seen": 2910519296
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.383245064997593e-05,
+      "loss": 1.258,
+      "theoretical_loss": 3.356979799806922,
+      "tokens_seen": 2910650368
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3824426255817685e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.3569682047498013,
+      "tokens_seen": 2910781440
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3816401861659445e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.356956610360979,
+      "tokens_seen": 2910912512
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3808377467501205e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.3569450166403865,
+      "tokens_seen": 2911043584
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.380035307334296e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3569334235879564,
+      "tokens_seen": 2911174656
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.379232867918472e-05,
+      "loss": 1.1682,
+      "theoretical_loss": 3.3569218312036186,
+      "tokens_seen": 2911305728
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.378430428502648e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3569102394873056,
+      "tokens_seen": 2911436800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.377627989086824e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.3568986484389485,
+      "tokens_seen": 2911567872
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3768255496709998e-05,
+      "loss": 1.2036,
+      "theoretical_loss": 3.3568870580584784,
+      "tokens_seen": 2911698944
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3760231102551758e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.3568754683458275,
+      "tokens_seen": 2911830016
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3752206708393518e-05,
+      "loss": 1.1561,
+      "theoretical_loss": 3.3568638793009264,
+      "tokens_seen": 2911961088
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.3744182314235277e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.3568522909237073,
+      "tokens_seen": 2912092160
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3736157920077034e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.3568407032141008,
+      "tokens_seen": 2912223232
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3728133525918794e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.3568291161720394,
+      "tokens_seen": 2912354304
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3720109131760554e-05,
+      "loss": 1.1626,
+      "theoretical_loss": 3.3568175297974543,
+      "tokens_seen": 2912485376
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.371208473760231e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.3568059440902767,
+      "tokens_seen": 2912616448
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.370406034344407e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.356794359050438,
+      "tokens_seen": 2912747520
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.369603594928583e-05,
+      "loss": 1.104,
+      "theoretical_loss": 3.3567827746778707,
+      "tokens_seen": 2912878592
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.368801155512759e-05,
+      "loss": 1.2344,
+      "theoretical_loss": 3.356771190972505,
+      "tokens_seen": 2913009664
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3679987160969346e-05,
+      "loss": 1.1694,
+      "theoretical_loss": 3.3567596079342734,
+      "tokens_seen": 2913140736
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3671962766811106e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.3567480255631077,
+      "tokens_seen": 2913271808
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3663938372652866e-05,
+      "loss": 1.1987,
+      "theoretical_loss": 3.356736443858938,
+      "tokens_seen": 2913402880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3655913978494626e-05,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3567248628216975,
+      "tokens_seen": 2913533952
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3647889584336383e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.356713282451317,
+      "tokens_seen": 2913665024
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3639865190178143e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.3567017027477286,
+      "tokens_seen": 2913796096
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3631840796019903e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.3566901237108633,
+      "tokens_seen": 2913927168
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.362381640186166e-05,
+      "loss": 1.3086,
+      "theoretical_loss": 3.3566785453406527,
+      "tokens_seen": 2914058240
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.361579200770342e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.356666967637029,
+      "tokens_seen": 2914189312
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.360776761354518e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.3566553905999235,
+      "tokens_seen": 2914320384
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.359974321938694e-05,
+      "loss": 1.2389,
+      "theoretical_loss": 3.356643814229268,
+      "tokens_seen": 2914451456
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3591718825228695e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3566322385249943,
+      "tokens_seen": 2914582528
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3583694431070455e-05,
+      "loss": 1.2676,
+      "theoretical_loss": 3.3566206634870337,
+      "tokens_seen": 2914713600
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 976720,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2558972835540771,
+      "objective/train/theoretical_loss": 3.3566090891153175,
+      "objective/train/tokens_used": 1285303776,
+      "theoretical_loss": 3.3566090891153175,
+      "tokens_seen": 2914844672
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3575670036912215e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3566090891153175,
+      "tokens_seen": 2914844672
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3567645642753975e-05,
+      "loss": 1.3013,
+      "theoretical_loss": 3.3565975154097787,
+      "tokens_seen": 2914975744
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.355962124859573e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.356585942370348,
+      "tokens_seen": 2915106816
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.355159685443749e-05,
+      "loss": 1.3217,
+      "theoretical_loss": 3.3565743699969572,
+      "tokens_seen": 2915237888
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.354357246027925e-05,
+      "loss": 1.1597,
+      "theoretical_loss": 3.3565627982895387,
+      "tokens_seen": 2915368960
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.353554806612101e-05,
+      "loss": 1.3019,
+      "theoretical_loss": 3.3565512272480236,
+      "tokens_seen": 2915500032
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3527523671962768e-05,
+      "loss": 1.1972,
+      "theoretical_loss": 3.3565396568723433,
+      "tokens_seen": 2915631104
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3519499277804528e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.35652808716243,
+      "tokens_seen": 2915762176
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3511474883646288e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.356516518118216,
+      "tokens_seen": 2915893248
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3503450489488044e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.3565049497396324,
+      "tokens_seen": 2916024320
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3495426095329804e-05,
+      "loss": 1.1969,
+      "theoretical_loss": 3.356493382026611,
+      "tokens_seen": 2916155392
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3487401701171564e-05,
+      "loss": 1.2489,
+      "theoretical_loss": 3.3564818149790843,
+      "tokens_seen": 2916286464
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3479377307013324e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3564702485969833,
+      "tokens_seen": 2916417536
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.347135291285508e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.35645868288024,
+      "tokens_seen": 2916548608
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.346332851869684e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.356447117828786,
+      "tokens_seen": 2916679680
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.34553041245386e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.356435553442554,
+      "tokens_seen": 2916810752
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.344727973038036e-05,
+      "loss": 1.2892,
+      "theoretical_loss": 3.356423989721475,
+      "tokens_seen": 2916941824
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3439255336222117e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.356412426665481,
+      "tokens_seen": 2917072896
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3431230942063876e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3564008642745042,
+      "tokens_seen": 2917203968
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3423206547905636e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.356389302548476,
+      "tokens_seen": 2917335040
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3415182153747393e-05,
+      "loss": 1.2892,
+      "theoretical_loss": 3.3563777414873286,
+      "tokens_seen": 2917466112
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3407157759589153e-05,
+      "loss": 1.221,
+      "theoretical_loss": 3.356366181090994,
+      "tokens_seen": 2917597184
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3399133365430913e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.3563546213594044,
+      "tokens_seen": 2917728256
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3391108971272673e-05,
+      "loss": 1.2358,
+      "theoretical_loss": 3.356343062292491,
+      "tokens_seen": 2917859328
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.338308457711443e-05,
+      "loss": 1.1742,
+      "theoretical_loss": 3.356331503890186,
+      "tokens_seen": 2917990400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.337506018295619e-05,
+      "loss": 1.1865,
+      "theoretical_loss": 3.3563199461524214,
+      "tokens_seen": 2918121472
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.336703578879795e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.3563083890791288,
+      "tokens_seen": 2918252544
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3359011394639705e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.3562968326702407,
+      "tokens_seen": 2918383616
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3350987000481465e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.356285276925689,
+      "tokens_seen": 2918514688
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3342962606323225e-05,
+      "loss": 1.2142,
+      "theoretical_loss": 3.356273721845405,
+      "tokens_seen": 2918645760
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3334938212164982e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.3562621674293216,
+      "tokens_seen": 2918776832
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3326913818006742e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.3562506136773704,
+      "tokens_seen": 2918907904
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.33188894238485e-05,
+      "loss": 1.2155,
+      "theoretical_loss": 3.356239060589483,
+      "tokens_seen": 2919038976
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3310865029690258e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.356227508165592,
+      "tokens_seen": 2919170048
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3302840635532018e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.3562159564056295,
+      "tokens_seen": 2919301120
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3294816241373778e-05,
+      "loss": 1.1423,
+      "theoretical_loss": 3.3562044053095272,
+      "tokens_seen": 2919432192
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3286791847215534e-05,
+      "loss": 1.1798,
+      "theoretical_loss": 3.356192854877217,
+      "tokens_seen": 2919563264
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3278767453057294e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.356181305108631,
+      "tokens_seen": 2919694336
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3270743058899054e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.356169756003702,
+      "tokens_seen": 2919825408
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.326271866474081e-05,
+      "loss": 1.1785,
+      "theoretical_loss": 3.3561582075623613,
+      "tokens_seen": 2919956480
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.325469427058257e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.356146659784541,
+      "tokens_seen": 2920087552
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.324666987642433e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3561351126701737,
+      "tokens_seen": 2920218624
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3238645482266087e-05,
+      "loss": 1.1608,
+      "theoretical_loss": 3.356123566219191,
+      "tokens_seen": 2920349696
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3230621088107847e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.3561120204315253,
+      "tokens_seen": 2920480768
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3222596693949607e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3561004753071084,
+      "tokens_seen": 2920611840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3214572299791367e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.356088930845873,
+      "tokens_seen": 2920742912
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3206547905633123e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3560773870477507,
+      "tokens_seen": 2920873984
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3198523511474883e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.356065843912674,
+      "tokens_seen": 2921005056
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3190499117316643e-05,
+      "loss": 1.1824,
+      "theoretical_loss": 3.356054301440575,
+      "tokens_seen": 2921136128
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3182474723158403e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.3560427596313858,
+      "tokens_seen": 2921267200
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 978874,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2121467590332031,
+      "objective/train/theoretical_loss": 3.3560312184850387,
+      "objective/train/tokens_used": 1291857376,
+      "theoretical_loss": 3.3560312184850387,
+      "tokens_seen": 2921398272
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.317445032900016e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3560312184850387,
+      "tokens_seen": 2921398272
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.316642593484192e-05,
+      "loss": 1.21,
+      "theoretical_loss": 3.356019678001466,
+      "tokens_seen": 2921529344
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.315840154068368e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3560081381805995,
+      "tokens_seen": 2921660416
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3150377146525436e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3559965990223715,
+      "tokens_seen": 2921791488
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3142352752367196e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.3559850605267147,
+      "tokens_seen": 2921922560
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3134328358208956e-05,
+      "loss": 1.1814,
+      "theoretical_loss": 3.3559735226935605,
+      "tokens_seen": 2922053632
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3126303964050716e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.3559619855228418,
+      "tokens_seen": 2922184704
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3118279569892472e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.3559504490144914,
+      "tokens_seen": 2922315776
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3110255175734232e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.35593891316844,
+      "tokens_seen": 2922446848
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3102230781575992e-05,
+      "loss": 1.234,
+      "theoretical_loss": 3.3559273779846213,
+      "tokens_seen": 2922577920
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3094206387417752e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.3559158434629666,
+      "tokens_seen": 2922708992
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.308618199325951e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.355904309603409,
+      "tokens_seen": 2922840064
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3078157599101268e-05,
+      "loss": 1.2457,
+      "theoretical_loss": 3.35589277640588,
+      "tokens_seen": 2922971136
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3070133204943028e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.3558812438703125,
+      "tokens_seen": 2923102208
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3062108810784785e-05,
+      "loss": 1.1497,
+      "theoretical_loss": 3.355869711996639,
+      "tokens_seen": 2923233280
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3054084416626545e-05,
+      "loss": 1.1229,
+      "theoretical_loss": 3.355858180784791,
+      "tokens_seen": 2923364352
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3046060022468305e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.355846650234702,
+      "tokens_seen": 2923495424
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.3038035628310064e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.3558351203463035,
+      "tokens_seen": 2923626496
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.303001123415182e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.355823591119528,
+      "tokens_seen": 2923757568
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.302198683999358e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.3558120625543078,
+      "tokens_seen": 2923888640
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.301396244583534e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3558005346505757,
+      "tokens_seen": 2924019712
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.30059380516771e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.355789007408264,
+      "tokens_seen": 2924150784
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2997913657518857e-05,
+      "loss": 1.1331,
+      "theoretical_loss": 3.3557774808273044,
+      "tokens_seen": 2924281856
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2989889263360617e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.35576595490763,
+      "tokens_seen": 2924412928
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2981864869202377e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.3557544296491733,
+      "tokens_seen": 2924544000
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2973840475044137e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.3557429050518666,
+      "tokens_seen": 2924675072
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2965816080885893e-05,
+      "loss": 1.1755,
+      "theoretical_loss": 3.355731381115642,
+      "tokens_seen": 2924806144
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2957791686727653e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.3557198578404326,
+      "tokens_seen": 2924937216
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2949767292569413e-05,
+      "loss": 1.3128,
+      "theoretical_loss": 3.3557083352261703,
+      "tokens_seen": 2925068288
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.294174289841117e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.355696813272788,
+      "tokens_seen": 2925199360
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.293371850425293e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.355685291980217,
+      "tokens_seen": 2925330432
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.292569411009469e-05,
+      "loss": 1.2072,
+      "theoretical_loss": 3.3556737713483917,
+      "tokens_seen": 2925461504
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.291766971593645e-05,
+      "loss": 1.2534,
+      "theoretical_loss": 3.3556622513772436,
+      "tokens_seen": 2925592576
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2909645321778206e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.355650732066705,
+      "tokens_seen": 2925723648
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2901620927619966e-05,
+      "loss": 1.2058,
+      "theoretical_loss": 3.355639213416709,
+      "tokens_seen": 2925854720
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2893596533461726e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.355627695427187,
+      "tokens_seen": 2925985792
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2885572139303486e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.355616178098073,
+      "tokens_seen": 2926116864
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2877547745145242e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.355604661429299,
+      "tokens_seen": 2926247936
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2869523350987002e-05,
+      "loss": 1.1968,
+      "theoretical_loss": 3.3555931454207975,
+      "tokens_seen": 2926379008
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2861498956828762e-05,
+      "loss": 1.2729,
+      "theoretical_loss": 3.3555816300725008,
+      "tokens_seen": 2926510080
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.285347456267052e-05,
+      "loss": 1.3345,
+      "theoretical_loss": 3.355570115384342,
+      "tokens_seen": 2926641152
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.284545016851228e-05,
+      "loss": 1.2436,
+      "theoretical_loss": 3.355558601356253,
+      "tokens_seen": 2926772224
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.283742577435404e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.3555470879881675,
+      "tokens_seen": 2926903296
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2829401380195798e-05,
+      "loss": 1.2608,
+      "theoretical_loss": 3.355535575280017,
+      "tokens_seen": 2927034368
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2821376986037555e-05,
+      "loss": 1.2548,
+      "theoretical_loss": 3.3555240632317345,
+      "tokens_seen": 2927165440
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2813352591879315e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.355512551843253,
+      "tokens_seen": 2927296512
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2805328197721075e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.355501041114505,
+      "tokens_seen": 2927427584
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2797303803562835e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3554895310454227,
+      "tokens_seen": 2927558656
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.278927940940459e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.355478021635939,
+      "tokens_seen": 2927689728
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.278125501524635e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.355466512885987,
+      "tokens_seen": 2927820800
+    },
+    {
+      "epoch": 0.77,
+      "objective/train/docs_used": 981126,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2067381143569946,
+      "objective/train/theoretical_loss": 3.3554550047954987,
+      "objective/train/tokens_used": 1298410976,
+      "theoretical_loss": 3.3554550047954987,
+      "tokens_seen": 2927951872
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.277323062108811e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.3554550047954987,
+      "tokens_seen": 2927951872
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2765206226929867e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.355443497364407,
+      "tokens_seen": 2928082944
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2757181832771627e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.3554319905926455,
+      "tokens_seen": 2928214016
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2749157438613387e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.3554204844801454,
+      "tokens_seen": 2928345088
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2741133044455147e-05,
+      "loss": 1.0985,
+      "theoretical_loss": 3.3554089790268407,
+      "tokens_seen": 2928476160
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.2733108650296904e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.355397474232663,
+      "tokens_seen": 2928607232
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2725084256138663e-05,
+      "loss": 1.3398,
+      "theoretical_loss": 3.355385970097546,
+      "tokens_seen": 2928738304
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2717059861980423e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.3553744666214222,
+      "tokens_seen": 2928869376
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2709035467822183e-05,
+      "loss": 1.2524,
+      "theoretical_loss": 3.355362963804224,
+      "tokens_seen": 2929000448
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.270101107366394e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.355351461645885,
+      "tokens_seen": 2929131520
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.26929866795057e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.3553399601463365,
+      "tokens_seen": 2929262592
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.268496228534746e-05,
+      "loss": 1.2452,
+      "theoretical_loss": 3.355328459305513,
+      "tokens_seen": 2929393664
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2676937891189216e-05,
+      "loss": 1.212,
+      "theoretical_loss": 3.355316959123346,
+      "tokens_seen": 2929524736
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2668913497030976e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.355305459599769,
+      "tokens_seen": 2929655808
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2660889102872736e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3552939607347145,
+      "tokens_seen": 2929786880
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2652864708714492e-05,
+      "loss": 1.2594,
+      "theoretical_loss": 3.3552824625281152,
+      "tokens_seen": 2929917952
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2644840314556252e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.355270964979905,
+      "tokens_seen": 2930049024
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2636815920398012e-05,
+      "loss": 1.2872,
+      "theoretical_loss": 3.3552594680900154,
+      "tokens_seen": 2930180096
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.262879152623977e-05,
+      "loss": 1.2485,
+      "theoretical_loss": 3.35524797185838,
+      "tokens_seen": 2930311168
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.262076713208153e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.355236476284931,
+      "tokens_seen": 2930442240
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.261274273792329e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.3552249813696027,
+      "tokens_seen": 2930573312
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2604718343765045e-05,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3552134871123265,
+      "tokens_seen": 2930704384
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2596693949606805e-05,
+      "loss": 1.2016,
+      "theoretical_loss": 3.355201993513036,
+      "tokens_seen": 2930835456
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2588669555448565e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.355190500571664,
+      "tokens_seen": 2930966528
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.258064516129032e-05,
+      "loss": 1.1974,
+      "theoretical_loss": 3.355179008288143,
+      "tokens_seen": 2931097600
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.257262076713208e-05,
+      "loss": 1.2219,
+      "theoretical_loss": 3.355167516662407,
+      "tokens_seen": 2931228672
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.256459637297384e-05,
+      "loss": 1.2586,
+      "theoretical_loss": 3.355156025694388,
+      "tokens_seen": 2931359744
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2556571978815598e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.355144535384019,
+      "tokens_seen": 2931490816
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2548547584657358e-05,
+      "loss": 1.233,
+      "theoretical_loss": 3.3551330457312334,
+      "tokens_seen": 2931621888
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2540523190499118e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.3551215567359636,
+      "tokens_seen": 2931752960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2532498796340878e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.3551100683981434,
+      "tokens_seen": 2931884032
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2524474402182634e-05,
+      "loss": 1.2462,
+      "theoretical_loss": 3.355098580717705,
+      "tokens_seen": 2932015104
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2516450008024394e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.355087093694582,
+      "tokens_seen": 2932146176
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2508425613866154e-05,
+      "loss": 1.1704,
+      "theoretical_loss": 3.3550756073287067,
+      "tokens_seen": 2932277248
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2500401219707914e-05,
+      "loss": 1.2544,
+      "theoretical_loss": 3.3550641216200128,
+      "tokens_seen": 2932408320
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.249237682554967e-05,
+      "loss": 1.2978,
+      "theoretical_loss": 3.3550526365684328,
+      "tokens_seen": 2932539392
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.248435243139143e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.3550411521739,
+      "tokens_seen": 2932670464
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.247632803723319e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.355029668436348,
+      "tokens_seen": 2932801536
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2468303643074947e-05,
+      "loss": 1.1763,
+      "theoretical_loss": 3.355018185355709,
+      "tokens_seen": 2932932608
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2460279248916707e-05,
+      "loss": 1.297,
+      "theoretical_loss": 3.355006702931916,
+      "tokens_seen": 2933063680
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2452254854758466e-05,
+      "loss": 1.1909,
+      "theoretical_loss": 3.354995221164903,
+      "tokens_seen": 2933194752
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2444230460600226e-05,
+      "loss": 1.2463,
+      "theoretical_loss": 3.3549837400546023,
+      "tokens_seen": 2933325824
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2436206066441983e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.354972259600947,
+      "tokens_seen": 2933456896
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2428181672283743e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.3549607798038705,
+      "tokens_seen": 2933587968
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2420157278125503e-05,
+      "loss": 1.1769,
+      "theoretical_loss": 3.354949300663306,
+      "tokens_seen": 2933719040
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2412132883967263e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.3549378221791866,
+      "tokens_seen": 2933850112
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.240410848980902e-05,
+      "loss": 1.1852,
+      "theoretical_loss": 3.3549263443514454,
+      "tokens_seen": 2933981184
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.239608409565078e-05,
+      "loss": 1.207,
+      "theoretical_loss": 3.3549148671800153,
+      "tokens_seen": 2934112256
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.238805970149254e-05,
+      "loss": 1.3335,
+      "theoretical_loss": 3.3549033906648296,
+      "tokens_seen": 2934243328
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2380035307334295e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.3548919148058216,
+      "tokens_seen": 2934374400
+    },
+    {
+      "epoch": 0.78,
+      "objective/train/docs_used": 983352,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.120585322380066,
+      "objective/train/theoretical_loss": 3.3548804396029244,
+      "objective/train/tokens_used": 1304964576,
+      "theoretical_loss": 3.3548804396029244,
+      "tokens_seen": 2934505472
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2372010913176055e-05,
+      "loss": 1.1589,
+      "theoretical_loss": 3.3548804396029244,
+      "tokens_seen": 2934505472
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2363986519017815e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.354868965056071,
+      "tokens_seen": 2934636544
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2355962124859575e-05,
+      "loss": 1.1954,
+      "theoretical_loss": 3.3548574911651947,
+      "tokens_seen": 2934767616
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.234793773070133e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3548460179302286,
+      "tokens_seen": 2934898688
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.233991333654309e-05,
+      "loss": 1.3126,
+      "theoretical_loss": 3.3548345453511064,
+      "tokens_seen": 2935029760
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.233188894238485e-05,
+      "loss": 1.2349,
+      "theoretical_loss": 3.3548230734277604,
+      "tokens_seen": 2935160832
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.232386454822661e-05,
+      "loss": 1.3378,
+      "theoretical_loss": 3.354811602160125,
+      "tokens_seen": 2935291904
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2315840154068368e-05,
+      "loss": 1.3503,
+      "theoretical_loss": 3.354800131548133,
+      "tokens_seen": 2935422976
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2307815759910128e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.354788661591717,
+      "tokens_seen": 2935554048
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2299791365751888e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.354777192290811,
+      "tokens_seen": 2935685120
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2291766971593644e-05,
+      "loss": 1.2925,
+      "theoretical_loss": 3.354765723645348,
+      "tokens_seen": 2935816192
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2283742577435404e-05,
+      "loss": 1.3202,
+      "theoretical_loss": 3.354754255655261,
+      "tokens_seen": 2935947264
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2275718183277164e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3547427883204843,
+      "tokens_seen": 2936078336
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2267693789118924e-05,
+      "loss": 1.2038,
+      "theoretical_loss": 3.35473132164095,
+      "tokens_seen": 2936209408
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.225966939496068e-05,
+      "loss": 1.283,
+      "theoretical_loss": 3.3547198556165925,
+      "tokens_seen": 2936340480
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.225164500080244e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.3547083902473442,
+      "tokens_seen": 2936471552
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.22436206066442e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.3546969255331387,
+      "tokens_seen": 2936602624
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.223559621248596e-05,
+      "loss": 1.3366,
+      "theoretical_loss": 3.3546854614739097,
+      "tokens_seen": 2936733696
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2227571818327717e-05,
+      "loss": 1.3545,
+      "theoretical_loss": 3.3546739980695897,
+      "tokens_seen": 2936864768
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2219547424169477e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.354662535320113,
+      "tokens_seen": 2936995840
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2211523030011236e-05,
+      "loss": 1.3234,
+      "theoretical_loss": 3.354651073225413,
+      "tokens_seen": 2937126912
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2203498635852993e-05,
+      "loss": 1.3113,
+      "theoretical_loss": 3.354639611785422,
+      "tokens_seen": 2937257984
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2195474241694753e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.3546281510000746,
+      "tokens_seen": 2937389056
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2187449847536513e-05,
+      "loss": 1.3248,
+      "theoretical_loss": 3.3546166908693036,
+      "tokens_seen": 2937520128
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2179425453378273e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.354605231393043,
+      "tokens_seen": 2937651200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.217140105922003e-05,
+      "loss": 1.3059,
+      "theoretical_loss": 3.3545937725712247,
+      "tokens_seen": 2937782272
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.216337666506179e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3545823144037836,
+      "tokens_seen": 2937913344
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.215535227090355e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.3545708568906525,
+      "tokens_seen": 2938044416
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.214732787674531e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.354559400031765,
+      "tokens_seen": 2938175488
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2139303482587065e-05,
+      "loss": 1.219,
+      "theoretical_loss": 3.354547943827055,
+      "tokens_seen": 2938306560
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2131279088428825e-05,
+      "loss": 1.3376,
+      "theoretical_loss": 3.3545364882764552,
+      "tokens_seen": 2938437632
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2123254694270585e-05,
+      "loss": 1.3216,
+      "theoretical_loss": 3.3545250333799,
+      "tokens_seen": 2938568704
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2115230300112342e-05,
+      "loss": 1.3224,
+      "theoretical_loss": 3.354513579137322,
+      "tokens_seen": 2938699776
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2107205905954102e-05,
+      "loss": 1.3144,
+      "theoretical_loss": 3.3545021255486547,
+      "tokens_seen": 2938830848
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.209918151179586e-05,
+      "loss": 1.3018,
+      "theoretical_loss": 3.354490672613832,
+      "tokens_seen": 2938961920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.209115711763762e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.3544792203327876,
+      "tokens_seen": 2939092992
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2083132723479378e-05,
+      "loss": 1.321,
+      "theoretical_loss": 3.3544677687054545,
+      "tokens_seen": 2939224064
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2075108329321138e-05,
+      "loss": 1.4213,
+      "theoretical_loss": 3.354456317731767,
+      "tokens_seen": 2939355136
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2067083935162898e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3544448674116576,
+      "tokens_seen": 2939486208
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2059059541004658e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.3544334177450605,
+      "tokens_seen": 2939617280
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2051035146846414e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3544219687319092,
+      "tokens_seen": 2939748352
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2043010752688174e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3544105203721375,
+      "tokens_seen": 2939879424
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2034986358529934e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.3543990726656787,
+      "tokens_seen": 2940010496
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.2026961964371694e-05,
+      "loss": 1.3345,
+      "theoretical_loss": 3.3543876256124663,
+      "tokens_seen": 2940141568
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.201893757021345e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.354376179212434,
+      "tokens_seen": 2940272640
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.201091317605521e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3543647334655153,
+      "tokens_seen": 2940403712
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.200288878189697e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.3543532883716445,
+      "tokens_seen": 2940534784
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1994864387738727e-05,
+      "loss": 1.3793,
+      "theoretical_loss": 3.354341843930754,
+      "tokens_seen": 2940665856
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1986839993580487e-05,
+      "loss": 1.3109,
+      "theoretical_loss": 3.3543304001427785,
+      "tokens_seen": 2940796928
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1978815599422247e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.354318957007651,
+      "tokens_seen": 2940928000
+    },
+    {
+      "epoch": 0.78,
+      "objective/train/docs_used": 985430,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1586353778839111,
+      "objective/train/theoretical_loss": 3.3543075145253063,
+      "objective/train/tokens_used": 1311518176,
+      "theoretical_loss": 3.3543075145253063,
+      "tokens_seen": 2941059072
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1970791205264003e-05,
+      "loss": 1.1892,
+      "theoretical_loss": 3.3543075145253063,
+      "tokens_seen": 2941059072
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1962766811105763e-05,
+      "loss": 1.3828,
+      "theoretical_loss": 3.3542960726956763,
+      "tokens_seen": 2941190144
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1954742416947523e-05,
+      "loss": 1.3045,
+      "theoretical_loss": 3.354284631518696,
+      "tokens_seen": 2941321216
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.194671802278928e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.3542731909942987,
+      "tokens_seen": 2941452288
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.193869362863104e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.354261751122418,
+      "tokens_seen": 2941583360
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.19306692344728e-05,
+      "loss": 1.2984,
+      "theoretical_loss": 3.354250311902988,
+      "tokens_seen": 2941714432
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1922644840314556e-05,
+      "loss": 1.218,
+      "theoretical_loss": 3.354238873335942,
+      "tokens_seen": 2941845504
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1914620446156316e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.3542274354212136,
+      "tokens_seen": 2941976576
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1906596051998076e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.354215998158737,
+      "tokens_seen": 2942107648
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1898571657839832e-05,
+      "loss": 1.2127,
+      "theoretical_loss": 3.3542045615484457,
+      "tokens_seen": 2942238720
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1890547263681592e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3541931255902737,
+      "tokens_seen": 2942369792
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1882522869523352e-05,
+      "loss": 1.2923,
+      "theoretical_loss": 3.3541816902841544,
+      "tokens_seen": 2942500864
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.187449847536511e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.354170255630022,
+      "tokens_seen": 2942631936
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.186647408120687e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.3541588216278098,
+      "tokens_seen": 2942763008
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.185844968704863e-05,
+      "loss": 1.1776,
+      "theoretical_loss": 3.3541473882774517,
+      "tokens_seen": 2942894080
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1850425292890388e-05,
+      "loss": 1.1595,
+      "theoretical_loss": 3.354135955578882,
+      "tokens_seen": 2943025152
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1842400898732145e-05,
+      "loss": 1.2887,
+      "theoretical_loss": 3.3541245235320343,
+      "tokens_seen": 2943156224
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1834376504573905e-05,
+      "loss": 1.3453,
+      "theoretical_loss": 3.354113092136842,
+      "tokens_seen": 2943287296
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1826352110415665e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3541016613932393,
+      "tokens_seen": 2943418368
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.181832771625742e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.35409023130116,
+      "tokens_seen": 2943549440
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.181030332209918e-05,
+      "loss": 1.2945,
+      "theoretical_loss": 3.3540788018605374,
+      "tokens_seen": 2943680512
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.180227892794094e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.354067373071307,
+      "tokens_seen": 2943811584
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.17942545337827e-05,
+      "loss": 1.2471,
+      "theoretical_loss": 3.3540559449334006,
+      "tokens_seen": 2943942656
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1786230139624457e-05,
+      "loss": 1.3034,
+      "theoretical_loss": 3.3540445174467535,
+      "tokens_seen": 2944073728
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1778205745466217e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.354033090611299,
+      "tokens_seen": 2944204800
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1770181351307977e-05,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3540216644269716,
+      "tokens_seen": 2944335872
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1762156957149737e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.354010238893704,
+      "tokens_seen": 2944466944
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1754132562991494e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.3539988140114314,
+      "tokens_seen": 2944598016
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1746108168833253e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3539873897800874,
+      "tokens_seen": 2944729088
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.1738083774675013e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.353975966199606,
+      "tokens_seen": 2944860160
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.173005938051677e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.35396454326992,
+      "tokens_seen": 2944991232
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.172203498635853e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.3539531209909645,
+      "tokens_seen": 2945122304
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.171401059220029e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.353941699362674,
+      "tokens_seen": 2945253376
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.170598619804205e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.353930278384981,
+      "tokens_seen": 2945384448
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1697961803883806e-05,
+      "loss": 1.3159,
+      "theoretical_loss": 3.3539188580578205,
+      "tokens_seen": 2945515520
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1689937409725566e-05,
+      "loss": 1.1513,
+      "theoretical_loss": 3.3539074383811265,
+      "tokens_seen": 2945646592
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1681913015567326e-05,
+      "loss": 1.336,
+      "theoretical_loss": 3.353896019354832,
+      "tokens_seen": 2945777664
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1673888621409086e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3538846009788728,
+      "tokens_seen": 2945908736
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1665864227250842e-05,
+      "loss": 1.1985,
+      "theoretical_loss": 3.353873183253181,
+      "tokens_seen": 2946039808
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1657839833092602e-05,
+      "loss": 1.1913,
+      "theoretical_loss": 3.353861766177692,
+      "tokens_seen": 2946170880
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1649815438934362e-05,
+      "loss": 1.3272,
+      "theoretical_loss": 3.3538503497523395,
+      "tokens_seen": 2946301952
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.164179104477612e-05,
+      "loss": 1.2006,
+      "theoretical_loss": 3.353838933977057,
+      "tokens_seen": 2946433024
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.163376665061788e-05,
+      "loss": 1.3695,
+      "theoretical_loss": 3.353827518851779,
+      "tokens_seen": 2946564096
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.162574225645964e-05,
+      "loss": 1.3244,
+      "theoretical_loss": 3.35381610437644,
+      "tokens_seen": 2946695168
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.16177178623014e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.3538046905509735,
+      "tokens_seen": 2946826240
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1609693468143155e-05,
+      "loss": 1.2238,
+      "theoretical_loss": 3.353793277375314,
+      "tokens_seen": 2946957312
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1601669073984915e-05,
+      "loss": 1.36,
+      "theoretical_loss": 3.353781864849395,
+      "tokens_seen": 2947088384
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1593644679826675e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.3537704529731514,
+      "tokens_seen": 2947219456
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1585620285668435e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.3537590417465166,
+      "tokens_seen": 2947350528
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.157759589151019e-05,
+      "loss": 1.2564,
+      "theoretical_loss": 3.353747631169425,
+      "tokens_seen": 2947481600
+    },
+    {
+      "epoch": 0.79,
+      "objective/train/docs_used": 987667,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2545603513717651,
+      "objective/train/theoretical_loss": 3.3537362212418107,
+      "objective/train/tokens_used": 1318071776,
+      "theoretical_loss": 3.3537362212418107,
+      "tokens_seen": 2947612672
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.156957149735195e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3537362212418107,
+      "tokens_seen": 2947612672
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.156154710319371e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.353724811963608,
+      "tokens_seen": 2947743744
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.155352270903547e-05,
+      "loss": 1.3773,
+      "theoretical_loss": 3.3537134033347513,
+      "tokens_seen": 2947874816
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1545498314877227e-05,
+      "loss": 1.3587,
+      "theoretical_loss": 3.3537019953551743,
+      "tokens_seen": 2948005888
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1537473920718987e-05,
+      "loss": 1.3155,
+      "theoretical_loss": 3.353690588024812,
+      "tokens_seen": 2948136960
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1529449526560747e-05,
+      "loss": 1.3139,
+      "theoretical_loss": 3.353679181343597,
+      "tokens_seen": 2948268032
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1521425132402504e-05,
+      "loss": 1.2672,
+      "theoretical_loss": 3.353667775311465,
+      "tokens_seen": 2948399104
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1513400738244264e-05,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3536563699283493,
+      "tokens_seen": 2948530176
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1505376344086024e-05,
+      "loss": 1.2621,
+      "theoretical_loss": 3.353644965194185,
+      "tokens_seen": 2948661248
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1497351949927783e-05,
+      "loss": 1.354,
+      "theoretical_loss": 3.3536335611089054,
+      "tokens_seen": 2948792320
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.148932755576954e-05,
+      "loss": 1.2427,
+      "theoretical_loss": 3.3536221576724454,
+      "tokens_seen": 2948923392
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.14813031616113e-05,
+      "loss": 1.275,
+      "theoretical_loss": 3.353610754884739,
+      "tokens_seen": 2949054464
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.147327876745306e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3535993527457206,
+      "tokens_seen": 2949185536
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.146525437329482e-05,
+      "loss": 1.2799,
+      "theoretical_loss": 3.3535879512553244,
+      "tokens_seen": 2949316608
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1457229979136576e-05,
+      "loss": 1.2616,
+      "theoretical_loss": 3.353576550413484,
+      "tokens_seen": 2949447680
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1449205584978336e-05,
+      "loss": 1.1488,
+      "theoretical_loss": 3.353565150220135,
+      "tokens_seen": 2949578752
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1441181190820096e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.3535537506752107,
+      "tokens_seen": 2949709824
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1433156796661853e-05,
+      "loss": 1.2719,
+      "theoretical_loss": 3.3535423517786462,
+      "tokens_seen": 2949840896
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1425132402503612e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.353530953530375,
+      "tokens_seen": 2949971968
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1417108008345372e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.3535195559303315,
+      "tokens_seen": 2950103040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1409083614187132e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.3535081589784506,
+      "tokens_seen": 2950234112
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.140105922002889e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3534967626746663,
+      "tokens_seen": 2950365184
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.139303482587065e-05,
+      "loss": 1.3266,
+      "theoretical_loss": 3.353485367018913,
+      "tokens_seen": 2950496256
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.138501043171241e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.3534739720111255,
+      "tokens_seen": 2950627328
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.137698603755417e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.3534625776512375,
+      "tokens_seen": 2950758400
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1368961643395925e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.3534511839391836,
+      "tokens_seen": 2950889472
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1360937249237685e-05,
+      "loss": 1.3487,
+      "theoretical_loss": 3.353439790874898,
+      "tokens_seen": 2951020544
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1352912855079445e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3534283984583153,
+      "tokens_seen": 2951151616
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.13448884609212e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3534170066893703,
+      "tokens_seen": 2951282688
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.133686406676296e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.353405615567997,
+      "tokens_seen": 2951413760
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.132883967260472e-05,
+      "loss": 1.2764,
+      "theoretical_loss": 3.3533942250941298,
+      "tokens_seen": 2951544832
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1320815278446478e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.3533828352677033,
+      "tokens_seen": 2951675904
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1312790884288238e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.353371446088652,
+      "tokens_seen": 2951806976
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1304766490129997e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.35336005755691,
+      "tokens_seen": 2951938048
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1296742095971754e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.353348669672412,
+      "tokens_seen": 2952069120
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1288717701813514e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.353337282435093,
+      "tokens_seen": 2952200192
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1280693307655274e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.3533258958448866,
+      "tokens_seen": 2952331264
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.127266891349703e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.3533145099017276,
+      "tokens_seen": 2952462336
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.126464451933879e-05,
+      "loss": 1.3474,
+      "theoretical_loss": 3.3533031246055507,
+      "tokens_seen": 2952593408
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.125662012518055e-05,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3532917399562905,
+      "tokens_seen": 2952724480
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1248595731022307e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.3532803559538813,
+      "tokens_seen": 2952855552
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1240571336864067e-05,
+      "loss": 1.267,
+      "theoretical_loss": 3.353268972598258,
+      "tokens_seen": 2952986624
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1232546942705826e-05,
+      "loss": 1.3051,
+      "theoretical_loss": 3.353257589889354,
+      "tokens_seen": 2953117696
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1224522548547583e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.3532462078271053,
+      "tokens_seen": 2953248768
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1216498154389343e-05,
+      "loss": 1.2865,
+      "theoretical_loss": 3.3532348264114455,
+      "tokens_seen": 2953379840
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1208473760231103e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.35322344564231,
+      "tokens_seen": 2953510912
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1200449366072863e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.3532120655196325,
+      "tokens_seen": 2953641984
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.119242497191462e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.353200686043348,
+      "tokens_seen": 2953773056
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.118440057775638e-05,
+      "loss": 1.2586,
+      "theoretical_loss": 3.3531893072133916,
+      "tokens_seen": 2953904128
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.117637618359814e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3531779290296972,
+      "tokens_seen": 2954035200
+    },
+    {
+      "epoch": 0.79,
+      "objective/train/docs_used": 989845,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3071619272232056,
+      "objective/train/theoretical_loss": 3.3531665514921993,
+      "objective/train/tokens_used": 1324625376,
+      "theoretical_loss": 3.3531665514921993,
+      "tokens_seen": 2954166272
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1168351789439896e-05,
+      "loss": 1.3217,
+      "theoretical_loss": 3.3531665514921993,
+      "tokens_seen": 2954166272
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1160327395281655e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.3531551746008335,
+      "tokens_seen": 2954297344
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1152303001123415e-05,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3531437983555334,
+      "tokens_seen": 2954428416
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1144278606965175e-05,
+      "loss": 1.3214,
+      "theoretical_loss": 3.353132422756234,
+      "tokens_seen": 2954559488
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1136254212806932e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.3531210478028703,
+      "tokens_seen": 2954690560
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.112822981864869e-05,
+      "loss": 1.3163,
+      "theoretical_loss": 3.3531096734953767,
+      "tokens_seen": 2954821632
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.112020542449045e-05,
+      "loss": 1.2812,
+      "theoretical_loss": 3.353098299833688,
+      "tokens_seen": 2954952704
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.111218103033221e-05,
+      "loss": 1.3276,
+      "theoretical_loss": 3.3530869268177383,
+      "tokens_seen": 2955083776
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1104156636173968e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3530755544474635,
+      "tokens_seen": 2955214848
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1096132242015728e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.3530641827227967,
+      "tokens_seen": 2955345920
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1088107847857488e-05,
+      "loss": 1.2949,
+      "theoretical_loss": 3.353052811643674,
+      "tokens_seen": 2955476992
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1080083453699248e-05,
+      "loss": 1.2173,
+      "theoretical_loss": 3.3530414412100296,
+      "tokens_seen": 2955608064
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1072059059541004e-05,
+      "loss": 1.149,
+      "theoretical_loss": 3.3530300714217987,
+      "tokens_seen": 2955739136
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1064034665382764e-05,
+      "loss": 1.3048,
+      "theoretical_loss": 3.3530187022789146,
+      "tokens_seen": 2955870208
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1056010271224524e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.353007333781314,
+      "tokens_seen": 2956001280
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.104798587706628e-05,
+      "loss": 1.2521,
+      "theoretical_loss": 3.3529959659289306,
+      "tokens_seen": 2956132352
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.103996148290804e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.352984598721699,
+      "tokens_seen": 2956263424
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.10319370887498e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.3529732321595542,
+      "tokens_seen": 2956394496
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.102391269459156e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.352961866242431,
+      "tokens_seen": 2956525568
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1015888300433317e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.352950500970265,
+      "tokens_seen": 2956656640
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.1007863906275077e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.35293913634299,
+      "tokens_seen": 2956787712
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0999839512116837e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.3529277723605406,
+      "tokens_seen": 2956918784
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0991815117958597e-05,
+      "loss": 1.3614,
+      "theoretical_loss": 3.352916409022853,
+      "tokens_seen": 2957049856
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0983790723800353e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3529050463298606,
+      "tokens_seen": 2957180928
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0975766329642113e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.352893684281499,
+      "tokens_seen": 2957312000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0967741935483873e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.352882322877703,
+      "tokens_seen": 2957443072
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.095971754132563e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.3528709621184074,
+      "tokens_seen": 2957574144
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.095169314716739e-05,
+      "loss": 1.3207,
+      "theoretical_loss": 3.352859602003547,
+      "tokens_seen": 2957705216
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.094366875300915e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.3528482425330566,
+      "tokens_seen": 2957836288
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.093564435885091e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.352836883706871,
+      "tokens_seen": 2957967360
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0927619964692666e-05,
+      "loss": 1.2973,
+      "theoretical_loss": 3.352825525524926,
+      "tokens_seen": 2958098432
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0919595570534426e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3528141679871557,
+      "tokens_seen": 2958229504
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0911571176376185e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.352802811093495,
+      "tokens_seen": 2958360576
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0903546782217945e-05,
+      "loss": 1.304,
+      "theoretical_loss": 3.3527914548438793,
+      "tokens_seen": 2958491648
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0895522388059702e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.352780099238243,
+      "tokens_seen": 2958622720
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0887497993901462e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3527687442765215,
+      "tokens_seen": 2958753792
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.087947359974322e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.35275738995865,
+      "tokens_seen": 2958884864
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0871449205584978e-05,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3527460362845627,
+      "tokens_seen": 2959015936
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0863424811426738e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.3527346832541944,
+      "tokens_seen": 2959147008
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0855400417268498e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.3527233308674815,
+      "tokens_seen": 2959278080
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0847376023110258e-05,
+      "loss": 1.2497,
+      "theoretical_loss": 3.352711979124358,
+      "tokens_seen": 2959409152
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0839351628952014e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3527006280247584,
+      "tokens_seen": 2959540224
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0831327234793774e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.352689277568619,
+      "tokens_seen": 2959671296
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0823302840635534e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.3526779277558743,
+      "tokens_seen": 2959802368
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0815278446477294e-05,
+      "loss": 1.1634,
+      "theoretical_loss": 3.352666578586459,
+      "tokens_seen": 2959933440
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.080725405231905e-05,
+      "loss": 1.3622,
+      "theoretical_loss": 3.352655230060308,
+      "tokens_seen": 2960064512
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.079922965816081e-05,
+      "loss": 1.2779,
+      "theoretical_loss": 3.3526438821773574,
+      "tokens_seen": 2960195584
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.079120526400257e-05,
+      "loss": 1.1855,
+      "theoretical_loss": 3.352632534937541,
+      "tokens_seen": 2960326656
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0783180869844327e-05,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3526211883407955,
+      "tokens_seen": 2960457728
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0775156475686087e-05,
+      "loss": 1.2605,
+      "theoretical_loss": 3.3526098423870545,
+      "tokens_seen": 2960588800
+    },
+    {
+      "debugging/Compilability": 0.08333333333333333,
+      "debugging/distinct-1-grams": 0.7149083490682447,
+      "debugging/entropy-1-grams": 5.232849655845552,
+      "debugging/length": 544.75,
+      "debugging/num_segments": 12,
+      "debugging/score": 0.05948124222167416,
+      "debugging/score_std": 0.02847918363119119,
+      "epoch": 0.79,
+      "objective/train/docs_used": 991929,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.4255846738815308,
+      "objective/train/theoretical_loss": 3.3525984970762535,
+      "objective/train/tokens_used": 1331178976,
+      "theoretical_loss": 3.3525984970762535,
+      "tokens_seen": 2960719872
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0767132081527847e-05,
+      "loss": 1.3499,
+      "theoretical_loss": 3.3525984970762535,
+      "tokens_seen": 2960719872
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0759107687369607e-05,
+      "loss": 1.3222,
+      "theoretical_loss": 3.3525871524083275,
+      "tokens_seen": 2960850944
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0751083293211363e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.352575808383212,
+      "tokens_seen": 2960982016
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0743058899053123e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.352564465000842,
+      "tokens_seen": 2961113088
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0735034504894883e-05,
+      "loss": 1.1966,
+      "theoretical_loss": 3.3525531222611527,
+      "tokens_seen": 2961244160
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.0727010110736643e-05,
+      "loss": 1.2377,
+      "theoretical_loss": 3.3525417801640787,
+      "tokens_seen": 2961375232
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.07189857165784e-05,
+      "loss": 1.1693,
+      "theoretical_loss": 3.352530438709556,
+      "tokens_seen": 2961506304
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.071096132242016e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.352519097897519,
+      "tokens_seen": 2961637376
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.070293692826192e-05,
+      "loss": 1.2649,
+      "theoretical_loss": 3.352507757727903,
+      "tokens_seen": 2961768448
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0694912534103676e-05,
+      "loss": 1.1591,
+      "theoretical_loss": 3.3524964182006443,
+      "tokens_seen": 2961899520
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0686888139945436e-05,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3524850793156764,
+      "tokens_seen": 2962030592
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0678863745787196e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3524737410729353,
+      "tokens_seen": 2962161664
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0670839351628955e-05,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3524624034723565,
+      "tokens_seen": 2962292736
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0662814957470712e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.3524510665138747,
+      "tokens_seen": 2962423808
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0654790563312472e-05,
+      "loss": 1.3041,
+      "theoretical_loss": 3.3524397301974256,
+      "tokens_seen": 2962554880
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0646766169154232e-05,
+      "loss": 1.2974,
+      "theoretical_loss": 3.352428394522944,
+      "tokens_seen": 2962685952
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.063874177499599e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.352417059490365,
+      "tokens_seen": 2962817024
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0630717380837748e-05,
+      "loss": 1.208,
+      "theoretical_loss": 3.3524057250996244,
+      "tokens_seen": 2962948096
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0622692986679508e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.352394391350657,
+      "tokens_seen": 2963079168
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0614668592521265e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.352383058243399,
+      "tokens_seen": 2963210240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0606644198363025e-05,
+      "loss": 1.2023,
+      "theoretical_loss": 3.3523717257777843,
+      "tokens_seen": 2963341312
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0598619804204784e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.3523603939537496,
+      "tokens_seen": 2963472384
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.059059541004654e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.352349062771229,
+      "tokens_seen": 2963603456
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.05825710158883e-05,
+      "loss": 1.2758,
+      "theoretical_loss": 3.352337732230158,
+      "tokens_seen": 2963734528
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.057454662173006e-05,
+      "loss": 1.2691,
+      "theoretical_loss": 3.3523264023304726,
+      "tokens_seen": 2963865600
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0566522227571817e-05,
+      "loss": 1.2146,
+      "theoretical_loss": 3.352315073072108,
+      "tokens_seen": 2963996672
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0558497833413577e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.352303744454999,
+      "tokens_seen": 2964127744
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0550473439255337e-05,
+      "loss": 1.2246,
+      "theoretical_loss": 3.3522924164790813,
+      "tokens_seen": 2964258816
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0542449045097094e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.3522810891442902,
+      "tokens_seen": 2964389888
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0534424650938854e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.352269762450561,
+      "tokens_seen": 2964520960
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0526400256780613e-05,
+      "loss": 1.2951,
+      "theoretical_loss": 3.3522584363978294,
+      "tokens_seen": 2964652032
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0518375862622373e-05,
+      "loss": 1.3202,
+      "theoretical_loss": 3.3522471109860303,
+      "tokens_seen": 2964783104
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.051035146846413e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3522357862150995,
+      "tokens_seen": 2964914176
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.050232707430589e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.352224462084972,
+      "tokens_seen": 2965045248
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.049430268014765e-05,
+      "loss": 1.3091,
+      "theoretical_loss": 3.352213138595584,
+      "tokens_seen": 2965176320
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0486278285989406e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.35220181574687,
+      "tokens_seen": 2965307392
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0478253891831166e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3521904935387656,
+      "tokens_seen": 2965438464
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0470229497672926e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.3521791719712066,
+      "tokens_seen": 2965569536
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0462205103514686e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3521678510441286,
+      "tokens_seen": 2965700608
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0454180709356442e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.352156530757467,
+      "tokens_seen": 2965831680
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0446156315198202e-05,
+      "loss": 1.304,
+      "theoretical_loss": 3.3521452111111567,
+      "tokens_seen": 2965962752
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0438131921039962e-05,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3521338921051336,
+      "tokens_seen": 2966093824
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0430107526881722e-05,
+      "loss": 1.1542,
+      "theoretical_loss": 3.352122573739333,
+      "tokens_seen": 2966224896
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.042208313272348e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3521112560136905,
+      "tokens_seen": 2966355968
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.041405873856524e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.352099938928142,
+      "tokens_seen": 2966487040
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0406034344407e-05,
+      "loss": 1.2172,
+      "theoretical_loss": 3.352088622482622,
+      "tokens_seen": 2966618112
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0398009950248755e-05,
+      "loss": 1.249,
+      "theoretical_loss": 3.3520773066770673,
+      "tokens_seen": 2966749184
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0389985556090515e-05,
+      "loss": 1.1674,
+      "theoretical_loss": 3.3520659915114126,
+      "tokens_seen": 2966880256
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0381961161932275e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3520546769855937,
+      "tokens_seen": 2967011328
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0373936767774035e-05,
+      "loss": 1.1407,
+      "theoretical_loss": 3.352043363099546,
+      "tokens_seen": 2967142400
+    },
+    {
+      "epoch": 0.8,
+      "objective/train/docs_used": 994007,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2186403274536133,
+      "objective/train/theoretical_loss": 3.3520320498532055,
+      "objective/train/tokens_used": 1337732576,
+      "theoretical_loss": 3.3520320498532055,
+      "tokens_seen": 2967273472
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.036591237361579e-05,
+      "loss": 1.1727,
+      "theoretical_loss": 3.3520320498532055,
+      "tokens_seen": 2967273472
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.035788797945755e-05,
+      "loss": 1.2128,
+      "theoretical_loss": 3.352020737246507,
+      "tokens_seen": 2967404544
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.034986358529931e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.3520094252793866,
+      "tokens_seen": 2967535616
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.034183919114107e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.35199811395178,
+      "tokens_seen": 2967666688
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0333814796982827e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.351986803263623,
+      "tokens_seen": 2967797760
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0325790402824587e-05,
+      "loss": 1.3206,
+      "theoretical_loss": 3.3519754932148507,
+      "tokens_seen": 2967928832
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0317766008666347e-05,
+      "loss": 1.2511,
+      "theoretical_loss": 3.351964183805398,
+      "tokens_seen": 2968059904
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0309741614508104e-05,
+      "loss": 1.2533,
+      "theoretical_loss": 3.351952875035203,
+      "tokens_seen": 2968190976
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0301717220349864e-05,
+      "loss": 1.2722,
+      "theoretical_loss": 3.3519415669041983,
+      "tokens_seen": 2968322048
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0293692826191624e-05,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3519302594123217,
+      "tokens_seen": 2968453120
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0285668432033384e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.351918952559508,
+      "tokens_seen": 2968584192
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.027764403787514e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.351907646345693,
+      "tokens_seen": 2968715264
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.02696196437169e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3518963407708124,
+      "tokens_seen": 2968846336
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.026159524955866e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.351885035834802,
+      "tokens_seen": 2968977408
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.025357085540042e-05,
+      "loss": 1.1813,
+      "theoretical_loss": 3.3518737315375975,
+      "tokens_seen": 2969108480
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0245546461242176e-05,
+      "loss": 1.2036,
+      "theoretical_loss": 3.3518624278791345,
+      "tokens_seen": 2969239552
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0237522067083936e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3518511248593486,
+      "tokens_seen": 2969370624
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0229497672925696e-05,
+      "loss": 1.177,
+      "theoretical_loss": 3.3518398224781754,
+      "tokens_seen": 2969501696
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0221473278767453e-05,
+      "loss": 1.2025,
+      "theoretical_loss": 3.351828520735551,
+      "tokens_seen": 2969632768
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0213448884609213e-05,
+      "loss": 1.264,
+      "theoretical_loss": 3.3518172196314113,
+      "tokens_seen": 2969763840
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0205424490450972e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.3518059191656913,
+      "tokens_seen": 2969894912
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0197400096292732e-05,
+      "loss": 1.2381,
+      "theoretical_loss": 3.3517946193383272,
+      "tokens_seen": 2970025984
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.018937570213449e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.3517833201492553,
+      "tokens_seen": 2970157056
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.018135130797625e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3517720215984106,
+      "tokens_seen": 2970288128
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.017332691381801e-05,
+      "loss": 1.1594,
+      "theoretical_loss": 3.351760723685729,
+      "tokens_seen": 2970419200
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.016530251965977e-05,
+      "loss": 1.1642,
+      "theoretical_loss": 3.3517494264111467,
+      "tokens_seen": 2970550272
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0157278125501525e-05,
+      "loss": 1.1772,
+      "theoretical_loss": 3.351738129774599,
+      "tokens_seen": 2970681344
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0149253731343285e-05,
+      "loss": 1.1732,
+      "theoretical_loss": 3.351726833776022,
+      "tokens_seen": 2970812416
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0141229337185045e-05,
+      "loss": 1.2026,
+      "theoretical_loss": 3.351715538415352,
+      "tokens_seen": 2970943488
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0133204943026805e-05,
+      "loss": 1.3047,
+      "theoretical_loss": 3.3517042436925237,
+      "tokens_seen": 2971074560
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.012518054886856e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.351692949607474,
+      "tokens_seen": 2971205632
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.011715615471032e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.3516816561601384,
+      "tokens_seen": 2971336704
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.010913176055208e-05,
+      "loss": 1.269,
+      "theoretical_loss": 3.3516703633504523,
+      "tokens_seen": 2971467776
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0101107366393838e-05,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3516590711783523,
+      "tokens_seen": 2971598848
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0093082972235598e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.351647779643774,
+      "tokens_seen": 2971729920
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0085058578077357e-05,
+      "loss": 1.2639,
+      "theoretical_loss": 3.351636488746653,
+      "tokens_seen": 2971860992
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0077034183919117e-05,
+      "loss": 1.2887,
+      "theoretical_loss": 3.3516251984869254,
+      "tokens_seen": 2971992064
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0069009789760874e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.351613908864527,
+      "tokens_seen": 2972123136
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0060985395602634e-05,
+      "loss": 1.1761,
+      "theoretical_loss": 3.3516026198793947,
+      "tokens_seen": 2972254208
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0052961001444394e-05,
+      "loss": 1.2262,
+      "theoretical_loss": 3.3515913315314627,
+      "tokens_seen": 2972385280
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0044936607286154e-05,
+      "loss": 1.1483,
+      "theoretical_loss": 3.351580043820668,
+      "tokens_seen": 2972516352
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.003691221312791e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.351568756746947,
+      "tokens_seen": 2972647424
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.002888781896967e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.351557470310235,
+      "tokens_seen": 2972778496
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.002086342481143e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3515461845104673,
+      "tokens_seen": 2972909568
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0012839030653186e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3515348993475813,
+      "tokens_seen": 2973040640
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.0004814636494946e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.3515236148215117,
+      "tokens_seen": 2973171712
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9996790242336706e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3515123309321955,
+      "tokens_seen": 2973302784
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9988765848178466e-05,
+      "loss": 1.2364,
+      "theoretical_loss": 3.3515010476795677,
+      "tokens_seen": 2973433856
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9980741454020223e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3514897650635653,
+      "tokens_seen": 2973564928
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9972717059861983e-05,
+      "loss": 1.1766,
+      "theoretical_loss": 3.351478483084124,
+      "tokens_seen": 2973696000
+    },
+    {
+      "epoch": 0.8,
+      "objective/train/docs_used": 996512,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.188795804977417,
+      "objective/train/theoretical_loss": 3.3514672017411797,
+      "objective/train/tokens_used": 1344286176,
+      "theoretical_loss": 3.3514672017411797,
+      "tokens_seen": 2973827072
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9964692665703743e-05,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3514672017411797,
+      "tokens_seen": 2973827072
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.99566682715455e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.351455921034668,
+      "tokens_seen": 2973958144
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.994864387738726e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3514446409645258,
+      "tokens_seen": 2974089216
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.994061948322902e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.351433361530689,
+      "tokens_seen": 2974220288
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9932595089070775e-05,
+      "loss": 1.2436,
+      "theoretical_loss": 3.351422082733093,
+      "tokens_seen": 2974351360
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9924570694912535e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3514108045716746,
+      "tokens_seen": 2974482432
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9916546300754295e-05,
+      "loss": 1.1442,
+      "theoretical_loss": 3.3513995270463695,
+      "tokens_seen": 2974613504
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.990852190659605e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.3513882501571137,
+      "tokens_seen": 2974744576
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.990049751243781e-05,
+      "loss": 1.1859,
+      "theoretical_loss": 3.351376973903844,
+      "tokens_seen": 2974875648
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.989247311827957e-05,
+      "loss": 1.2467,
+      "theoretical_loss": 3.3513656982864957,
+      "tokens_seen": 2975006720
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9884448724121328e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.351354423305005,
+      "tokens_seen": 2975137792
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9876424329963088e-05,
+      "loss": 1.2237,
+      "theoretical_loss": 3.351343148959309,
+      "tokens_seen": 2975268864
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9868399935804848e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.3513318752493424,
+      "tokens_seen": 2975399936
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9860375541646604e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.351320602175042,
+      "tokens_seen": 2975531008
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9852351147488364e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3513093297363445,
+      "tokens_seen": 2975662080
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9844326753330124e-05,
+      "loss": 1.2744,
+      "theoretical_loss": 3.3512980579331852,
+      "tokens_seen": 2975793152
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.983630235917188e-05,
+      "loss": 1.1789,
+      "theoretical_loss": 3.351286786765501,
+      "tokens_seen": 2975924224
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.982827796501364e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.3512755162332275,
+      "tokens_seen": 2976055296
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.98202535708554e-05,
+      "loss": 1.2194,
+      "theoretical_loss": 3.3512642463363016,
+      "tokens_seen": 2976186368
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.981222917669716e-05,
+      "loss": 1.1806,
+      "theoretical_loss": 3.3512529770746586,
+      "tokens_seen": 2976317440
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9804204782538917e-05,
+      "loss": 1.192,
+      "theoretical_loss": 3.351241708448235,
+      "tokens_seen": 2976448512
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9796180388380677e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3512304404569675,
+      "tokens_seen": 2976579584
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9788155994222437e-05,
+      "loss": 1.248,
+      "theoretical_loss": 3.351219173100792,
+      "tokens_seen": 2976710656
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9780131600064197e-05,
+      "loss": 1.1456,
+      "theoretical_loss": 3.3512079063796447,
+      "tokens_seen": 2976841728
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9772107205905953e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.3511966402934625,
+      "tokens_seen": 2976972800
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9764082811747713e-05,
+      "loss": 1.1334,
+      "theoretical_loss": 3.3511853748421805,
+      "tokens_seen": 2977103872
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9756058417589473e-05,
+      "loss": 1.213,
+      "theoretical_loss": 3.3511741100257355,
+      "tokens_seen": 2977234944
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.974803402343123e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.3511628458440637,
+      "tokens_seen": 2977366016
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.974000962927299e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.351151582297102,
+      "tokens_seen": 2977497088
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.973198523511475e-05,
+      "loss": 1.197,
+      "theoretical_loss": 3.3511403193847857,
+      "tokens_seen": 2977628160
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.972396084095651e-05,
+      "loss": 1.227,
+      "theoretical_loss": 3.3511290571070522,
+      "tokens_seen": 2977759232
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9715936446798266e-05,
+      "loss": 1.2072,
+      "theoretical_loss": 3.351117795463837,
+      "tokens_seen": 2977890304
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9707912052640026e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3511065344550763,
+      "tokens_seen": 2978021376
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.9699887658481786e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.3510952740807074,
+      "tokens_seen": 2978152448
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9691863264323545e-05,
+      "loss": 1.255,
+      "theoretical_loss": 3.3510840143406657,
+      "tokens_seen": 2978283520
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9683838870165302e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.351072755234888,
+      "tokens_seen": 2978414592
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9675814476007062e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.3510614967633106,
+      "tokens_seen": 2978545664
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9667790081848822e-05,
+      "loss": 1.178,
+      "theoretical_loss": 3.3510502389258696,
+      "tokens_seen": 2978676736
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.965976568769058e-05,
+      "loss": 1.1998,
+      "theoretical_loss": 3.3510389817225015,
+      "tokens_seen": 2978807808
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9651741293532338e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.351027725153143,
+      "tokens_seen": 2978938880
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9643716899374098e-05,
+      "loss": 1.1736,
+      "theoretical_loss": 3.3510164692177304,
+      "tokens_seen": 2979069952
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9635692505215858e-05,
+      "loss": 1.2926,
+      "theoretical_loss": 3.3510052139162,
+      "tokens_seen": 2979201024
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9627668111057615e-05,
+      "loss": 1.2804,
+      "theoretical_loss": 3.350993959248488,
+      "tokens_seen": 2979332096
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9619643716899374e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.3509827052145313,
+      "tokens_seen": 2979463168
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9611619322741134e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.350971451814266,
+      "tokens_seen": 2979594240
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9603594928582894e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.3509601990476288,
+      "tokens_seen": 2979725312
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.959557053442465e-05,
+      "loss": 1.25,
+      "theoretical_loss": 3.3509489469145555,
+      "tokens_seen": 2979856384
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.958754614026641e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.3509376954149834,
+      "tokens_seen": 2979987456
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.957952174610817e-05,
+      "loss": 1.2556,
+      "theoretical_loss": 3.3509264445488487,
+      "tokens_seen": 2980118528
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.957149735194993e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3509151943160873,
+      "tokens_seen": 2980249600
+    },
+    {
+      "epoch": 0.81,
+      "objective/train/docs_used": 998650,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3528218269348145,
+      "objective/train/theoretical_loss": 3.3509039447166367,
+      "objective/train/tokens_used": 1350839776,
+      "theoretical_loss": 3.3509039447166367,
+      "tokens_seen": 2980380672
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9563472957791687e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.3509039447166367,
+      "tokens_seen": 2980380672
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9555448563633447e-05,
+      "loss": 1.231,
+      "theoretical_loss": 3.350892695750433,
+      "tokens_seen": 2980511744
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9547424169475207e-05,
+      "loss": 1.1388,
+      "theoretical_loss": 3.3508814474174122,
+      "tokens_seen": 2980642816
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9539399775316963e-05,
+      "loss": 1.2933,
+      "theoretical_loss": 3.3508701997175114,
+      "tokens_seen": 2980773888
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9531375381158723e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3508589526506665,
+      "tokens_seen": 2980904960
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9523350987000483e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.3508477062168147,
+      "tokens_seen": 2981036032
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9515326592842243e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3508364604158927,
+      "tokens_seen": 2981167104
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9507302198684e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.3508252152478364,
+      "tokens_seen": 2981298176
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.949927780452576e-05,
+      "loss": 1.3354,
+      "theoretical_loss": 3.350813970712583,
+      "tokens_seen": 2981429248
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.949125341036752e-05,
+      "loss": 1.1749,
+      "theoretical_loss": 3.350802726810068,
+      "tokens_seen": 2981560320
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.948322901620928e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.3507914835402293,
+      "tokens_seen": 2981691392
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9475204622051036e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3507802409030023,
+      "tokens_seen": 2981822464
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9467180227892796e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.3507689988983245,
+      "tokens_seen": 2981953536
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9459155833734556e-05,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3507577575261323,
+      "tokens_seen": 2982084608
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9451131439576312e-05,
+      "loss": 1.1654,
+      "theoretical_loss": 3.350746516786362,
+      "tokens_seen": 2982215680
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9443107045418072e-05,
+      "loss": 1.259,
+      "theoretical_loss": 3.3507352766789507,
+      "tokens_seen": 2982346752
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9435082651259832e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3507240372038347,
+      "tokens_seen": 2982477824
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9427058257101592e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.3507127983609504,
+      "tokens_seen": 2982608896
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.941903386294335e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.3507015601502355,
+      "tokens_seen": 2982739968
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9411009468785108e-05,
+      "loss": 1.2538,
+      "theoretical_loss": 3.350690322571625,
+      "tokens_seen": 2982871040
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9402985074626868e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.3506790856250572,
+      "tokens_seen": 2983002112
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9394960680468628e-05,
+      "loss": 1.1787,
+      "theoretical_loss": 3.350667849310468,
+      "tokens_seen": 2983133184
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9386936286310385e-05,
+      "loss": 1.2601,
+      "theoretical_loss": 3.3506566136277938,
+      "tokens_seen": 2983264256
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9378911892152144e-05,
+      "loss": 1.1506,
+      "theoretical_loss": 3.350645378576972,
+      "tokens_seen": 2983395328
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9370887497993904e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.3506341441579393,
+      "tokens_seen": 2983526400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.936286310383566e-05,
+      "loss": 1.1737,
+      "theoretical_loss": 3.3506229103706318,
+      "tokens_seen": 2983657472
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.935483870967742e-05,
+      "loss": 1.1592,
+      "theoretical_loss": 3.350611677214986,
+      "tokens_seen": 2983788544
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.934681431551918e-05,
+      "loss": 1.2972,
+      "theoretical_loss": 3.35060044469094,
+      "tokens_seen": 2983919616
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.933878992136094e-05,
+      "loss": 1.2705,
+      "theoretical_loss": 3.3505892127984294,
+      "tokens_seen": 2984050688
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9330765527202697e-05,
+      "loss": 1.1875,
+      "theoretical_loss": 3.3505779815373913,
+      "tokens_seen": 2984181760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9322741133044457e-05,
+      "loss": 1.2391,
+      "theoretical_loss": 3.3505667509077623,
+      "tokens_seen": 2984312832
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9314716738886217e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.3505555209094795,
+      "tokens_seen": 2984443904
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9306692344727973e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.3505442915424797,
+      "tokens_seen": 2984574976
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9298667950569733e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.350533062806699,
+      "tokens_seen": 2984706048
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9290643556411493e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.350521834702075,
+      "tokens_seen": 2984837120
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.928261916225325e-05,
+      "loss": 1.2502,
+      "theoretical_loss": 3.350510607228544,
+      "tokens_seen": 2984968192
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.927459476809501e-05,
+      "loss": 1.2758,
+      "theoretical_loss": 3.3504993803860437,
+      "tokens_seen": 2985099264
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.926657037393677e-05,
+      "loss": 1.292,
+      "theoretical_loss": 3.3504881541745095,
+      "tokens_seen": 2985230336
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9258545979778526e-05,
+      "loss": 1.1861,
+      "theoretical_loss": 3.350476928593879,
+      "tokens_seen": 2985361408
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9250521585620286e-05,
+      "loss": 1.2947,
+      "theoretical_loss": 3.3504657036440895,
+      "tokens_seen": 2985492480
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9242497191462046e-05,
+      "loss": 1.254,
+      "theoretical_loss": 3.3504544793250775,
+      "tokens_seen": 2985623552
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9234472797303802e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.350443255636779,
+      "tokens_seen": 2985754624
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9226448403145562e-05,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3504320325791324,
+      "tokens_seen": 2985885696
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9218424008987322e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.350420810152073,
+      "tokens_seen": 2986016768
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.921039961482908e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3504095883555394,
+      "tokens_seen": 2986147840
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.920237522067084e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.350398367189467,
+      "tokens_seen": 2986278912
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.91943508265126e-05,
+      "loss": 1.1978,
+      "theoretical_loss": 3.3503871466537936,
+      "tokens_seen": 2986409984
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9186326432354355e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3503759267484554,
+      "tokens_seen": 2986541056
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9178302038196115e-05,
+      "loss": 1.2185,
+      "theoretical_loss": 3.3503647074733904,
+      "tokens_seen": 2986672128
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9170277644037875e-05,
+      "loss": 1.3018,
+      "theoretical_loss": 3.3503534888285347,
+      "tokens_seen": 2986803200
+    },
+    {
+      "epoch": 0.81,
+      "objective/train/docs_used": 1000654,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3021142482757568,
+      "objective/train/theoretical_loss": 3.350342270813825,
+      "objective/train/tokens_used": 1357393376,
+      "theoretical_loss": 3.350342270813825,
+      "tokens_seen": 2986934272
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9162253249879635e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.350342270813825,
+      "tokens_seen": 2986934272
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.915422885572139e-05,
+      "loss": 1.154,
+      "theoretical_loss": 3.3503310534291995,
+      "tokens_seen": 2987065344
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.914620446156315e-05,
+      "loss": 1.2528,
+      "theoretical_loss": 3.3503198366745934,
+      "tokens_seen": 2987196416
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.913818006740491e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.3503086205499453,
+      "tokens_seen": 2987327488
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.913015567324667e-05,
+      "loss": 1.167,
+      "theoretical_loss": 3.3502974050551915,
+      "tokens_seen": 2987458560
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9122131279088428e-05,
+      "loss": 1.1403,
+      "theoretical_loss": 3.3502861901902685,
+      "tokens_seen": 2987589632
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9114106884930188e-05,
+      "loss": 1.2331,
+      "theoretical_loss": 3.3502749759551147,
+      "tokens_seen": 2987720704
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9106082490771947e-05,
+      "loss": 1.2737,
+      "theoretical_loss": 3.3502637623496656,
+      "tokens_seen": 2987851776
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9098058096613707e-05,
+      "loss": 1.1881,
+      "theoretical_loss": 3.3502525493738586,
+      "tokens_seen": 2987982848
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9090033702455464e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.3502413370276316,
+      "tokens_seen": 2988113920
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9082009308297224e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.3502301253109206,
+      "tokens_seen": 2988244992
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9073984914138984e-05,
+      "loss": 1.246,
+      "theoretical_loss": 3.3502189142236634,
+      "tokens_seen": 2988376064
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.906596051998074e-05,
+      "loss": 1.1675,
+      "theoretical_loss": 3.3502077037657965,
+      "tokens_seen": 2988507136
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.90579361258225e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.350196493937257,
+      "tokens_seen": 2988638208
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.904991173166426e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3501852847379823,
+      "tokens_seen": 2988769280
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.904188733750602e-05,
+      "loss": 1.1423,
+      "theoretical_loss": 3.3501740761679093,
+      "tokens_seen": 2988900352
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9033862943347776e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3501628682269753,
+      "tokens_seen": 2989031424
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9025838549189536e-05,
+      "loss": 1.216,
+      "theoretical_loss": 3.3501516609151167,
+      "tokens_seen": 2989162496
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9017814155031296e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.350140454232272,
+      "tokens_seen": 2989293568
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9009789760873056e-05,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3501292481783764,
+      "tokens_seen": 2989424640
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9001765366714813e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.350118042753369,
+      "tokens_seen": 2989555712
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8993740972556573e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.3501068379571857,
+      "tokens_seen": 2989686784
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8985716578398332e-05,
+      "loss": 1.1509,
+      "theoretical_loss": 3.350095633789764,
+      "tokens_seen": 2989817856
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.897769218424009e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.350084430251041,
+      "tokens_seen": 2989948928
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.896966779008185e-05,
+      "loss": 1.286,
+      "theoretical_loss": 3.3500732273409533,
+      "tokens_seen": 2990080000
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.896164339592361e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.3500620250594393,
+      "tokens_seen": 2990211072
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.895361900176537e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.350050823406435,
+      "tokens_seen": 2990342144
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8945594607607125e-05,
+      "loss": 1.2183,
+      "theoretical_loss": 3.3500396223818787,
+      "tokens_seen": 2990473216
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8937570213448885e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.350028421985707,
+      "tokens_seen": 2990604288
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8929545819290645e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.350017222217857,
+      "tokens_seen": 2990735360
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8921521425132405e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3500060230782656,
+      "tokens_seen": 2990866432
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.891349703097416e-05,
+      "loss": 1.2847,
+      "theoretical_loss": 3.349994824566871,
+      "tokens_seen": 2990997504
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.890547263681592e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.34998362668361,
+      "tokens_seen": 2991128576
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.889744824265768e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.349972429428419,
+      "tokens_seen": 2991259648
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8889423848499438e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3499612328012365,
+      "tokens_seen": 2991390720
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8881399454341198e-05,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3499500368019994,
+      "tokens_seen": 2991521792
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8873375060182958e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3499388414306446,
+      "tokens_seen": 2991652864
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8865350666024717e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.3499276466871097,
+      "tokens_seen": 2991783936
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8857326271866474e-05,
+      "loss": 1.2342,
+      "theoretical_loss": 3.349916452571332,
+      "tokens_seen": 2991915008
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8849301877708234e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.3499052590832488,
+      "tokens_seen": 2992046080
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8841277483549994e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.349894066222797,
+      "tokens_seen": 2992177152
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8833253089391754e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.3498828739899142,
+      "tokens_seen": 2992308224
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.882522869523351e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.3498716823845385,
+      "tokens_seen": 2992439296
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.881720430107527e-05,
+      "loss": 1.1858,
+      "theoretical_loss": 3.3498604914066057,
+      "tokens_seen": 2992570368
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.880917990691703e-05,
+      "loss": 1.2348,
+      "theoretical_loss": 3.349849301056054,
+      "tokens_seen": 2992701440
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8801155512758787e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.3498381113328213,
+      "tokens_seen": 2992832512
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8793131118600546e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.3498269222368435,
+      "tokens_seen": 2992963584
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8785106724442306e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.3498157337680596,
+      "tokens_seen": 2993094656
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8777082330284066e-05,
+      "loss": 1.1884,
+      "theoretical_loss": 3.3498045459264056,
+      "tokens_seen": 2993225728
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8769057936125823e-05,
+      "loss": 1.2775,
+      "theoretical_loss": 3.3497933587118203,
+      "tokens_seen": 2993356800
+    },
+    {
+      "epoch": 0.81,
+      "objective/train/docs_used": 1002769,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1278400421142578,
+      "objective/train/theoretical_loss": 3.3497821721242396,
+      "objective/train/tokens_used": 1363946976,
+      "theoretical_loss": 3.3497821721242396,
+      "tokens_seen": 2993487872
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8761033541967583e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.3497821721242396,
+      "tokens_seen": 2993487872
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8753009147809343e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.349770986163602,
+      "tokens_seen": 2993618944
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8744984753651103e-05,
+      "loss": 1.1984,
+      "theoretical_loss": 3.349759800829844,
+      "tokens_seen": 2993750016
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.873696035949286e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.3497486161229038,
+      "tokens_seen": 2993881088
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.872893596533462e-05,
+      "loss": 1.2558,
+      "theoretical_loss": 3.3497374320427187,
+      "tokens_seen": 2994012160
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.872091157117638e-05,
+      "loss": 1.2749,
+      "theoretical_loss": 3.3497262485892256,
+      "tokens_seen": 2994143232
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8712887177018135e-05,
+      "loss": 1.2311,
+      "theoretical_loss": 3.349715065762363,
+      "tokens_seen": 2994274304
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8704862782859895e-05,
+      "loss": 1.1842,
+      "theoretical_loss": 3.3497038835620674,
+      "tokens_seen": 2994405376
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8696838388701655e-05,
+      "loss": 1.2816,
+      "theoretical_loss": 3.3496927019882765,
+      "tokens_seen": 2994536448
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.8688813994543415e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.349681521040928,
+      "tokens_seen": 2994667520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.868078960038517e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.3496703407199595,
+      "tokens_seen": 2994798592
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.867276520622693e-05,
+      "loss": 1.3002,
+      "theoretical_loss": 3.3496591610253077,
+      "tokens_seen": 2994929664
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.866474081206869e-05,
+      "loss": 1.2348,
+      "theoretical_loss": 3.3496479819569114,
+      "tokens_seen": 2995060736
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.865671641791045e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.3496368035147066,
+      "tokens_seen": 2995191808
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8648692023752208e-05,
+      "loss": 1.314,
+      "theoretical_loss": 3.349625625698632,
+      "tokens_seen": 2995322880
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8640667629593968e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3496144485086248,
+      "tokens_seen": 2995453952
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8632643235435728e-05,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3496032719446225,
+      "tokens_seen": 2995585024
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8624618841277484e-05,
+      "loss": 1.3001,
+      "theoretical_loss": 3.3495920960065626,
+      "tokens_seen": 2995716096
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8616594447119244e-05,
+      "loss": 1.1962,
+      "theoretical_loss": 3.3495809206943825,
+      "tokens_seen": 2995847168
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8608570052961004e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.3495697460080205,
+      "tokens_seen": 2995978240
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.860054565880276e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.349558571947413,
+      "tokens_seen": 2996109312
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.859252126464452e-05,
+      "loss": 1.2089,
+      "theoretical_loss": 3.3495473985124984,
+      "tokens_seen": 2996240384
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.858449687048628e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3495362257032144,
+      "tokens_seen": 2996371456
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8576472476328037e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.349525053519498,
+      "tokens_seen": 2996502528
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8568448082169797e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.349513881961287,
+      "tokens_seen": 2996633600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8560423688011557e-05,
+      "loss": 1.2429,
+      "theoretical_loss": 3.34950271102852,
+      "tokens_seen": 2996764672
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8552399293853313e-05,
+      "loss": 1.2348,
+      "theoretical_loss": 3.3494915407211328,
+      "tokens_seen": 2996895744
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8544374899695073e-05,
+      "loss": 1.2808,
+      "theoretical_loss": 3.3494803710390646,
+      "tokens_seen": 2997026816
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8536350505536833e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3494692019822523,
+      "tokens_seen": 2997157888
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.852832611137859e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.3494580335506337,
+      "tokens_seen": 2997288960
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.852030171722035e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.3494468657441465,
+      "tokens_seen": 2997420032
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.851227732306211e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3494356985627283,
+      "tokens_seen": 2997551104
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8504252928903866e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.349424532006317,
+      "tokens_seen": 2997682176
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8496228534745626e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.34941336607485,
+      "tokens_seen": 2997813248
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8488204140587386e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3494022007682656,
+      "tokens_seen": 2997944320
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8480179746429146e-05,
+      "loss": 1.1838,
+      "theoretical_loss": 3.3493910360865007,
+      "tokens_seen": 2998075392
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8472155352270902e-05,
+      "loss": 1.2465,
+      "theoretical_loss": 3.3493798720294934,
+      "tokens_seen": 2998206464
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8464130958112662e-05,
+      "loss": 1.2671,
+      "theoretical_loss": 3.3493687085971815,
+      "tokens_seen": 2998337536
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8456106563954422e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.349357545789503,
+      "tokens_seen": 2998468608
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8448082169796182e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.349346383606395,
+      "tokens_seen": 2998599680
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8440057775637938e-05,
+      "loss": 1.1781,
+      "theoretical_loss": 3.349335222047795,
+      "tokens_seen": 2998730752
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8432033381479698e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.3493240611136423,
+      "tokens_seen": 2998861824
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8424008987321458e-05,
+      "loss": 1.308,
+      "theoretical_loss": 3.349312900803873,
+      "tokens_seen": 2998992896
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8415984593163215e-05,
+      "loss": 1.2205,
+      "theoretical_loss": 3.349301741118426,
+      "tokens_seen": 2999123968
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8407960199004975e-05,
+      "loss": 1.2234,
+      "theoretical_loss": 3.3492905820572387,
+      "tokens_seen": 2999255040
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8399935804846734e-05,
+      "loss": 1.2067,
+      "theoretical_loss": 3.3492794236202488,
+      "tokens_seen": 2999386112
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8391911410688494e-05,
+      "loss": 1.167,
+      "theoretical_loss": 3.349268265807394,
+      "tokens_seen": 2999517184
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.838388701653025e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.3492571086186125,
+      "tokens_seen": 2999648256
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.837586262237201e-05,
+      "loss": 1.2055,
+      "theoretical_loss": 3.349245952053842,
+      "tokens_seen": 2999779328
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.836783822821377e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.34923479611302,
+      "tokens_seen": 2999910400
+    },
+    {
+      "epoch": 0.82,
+      "objective/train/docs_used": 1004751,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2346285581588745,
+      "objective/train/theoretical_loss": 3.3492236407960845,
+      "objective/train/tokens_used": 1370500576,
+      "theoretical_loss": 3.3492236407960845,
+      "tokens_seen": 3000041472
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.835981383405553e-05,
+      "loss": 1.2254,
+      "theoretical_loss": 3.3492236407960845,
+      "tokens_seen": 3000041472
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8351789439897287e-05,
+      "loss": 1.1561,
+      "theoretical_loss": 3.3492124861029735,
+      "tokens_seen": 3000172544
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8343765045739047e-05,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3492013320336254,
+      "tokens_seen": 3000303616
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8335740651580807e-05,
+      "loss": 1.2758,
+      "theoretical_loss": 3.3491901785879774,
+      "tokens_seen": 3000434688
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8327716257422563e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.349179025765967,
+      "tokens_seen": 3000565760
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8319691863264323e-05,
+      "loss": 1.209,
+      "theoretical_loss": 3.3491678735675325,
+      "tokens_seen": 3000696832
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8311667469106083e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.3491567219926126,
+      "tokens_seen": 3000827904
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8303643074947843e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.349145571041144,
+      "tokens_seen": 3000958976
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.82956186807896e-05,
+      "loss": 1.2546,
+      "theoretical_loss": 3.349134420713065,
+      "tokens_seen": 3001090048
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.828759428663136e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.349123271008314,
+      "tokens_seen": 3001221120
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.827956989247312e-05,
+      "loss": 1.1894,
+      "theoretical_loss": 3.3491121219268285,
+      "tokens_seen": 3001352192
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.827154549831488e-05,
+      "loss": 1.2678,
+      "theoretical_loss": 3.349100973468546,
+      "tokens_seen": 3001483264
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8263521104156636e-05,
+      "loss": 1.2052,
+      "theoretical_loss": 3.3490898256334054,
+      "tokens_seen": 3001614336
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8255496709998396e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3490786784213444,
+      "tokens_seen": 3001745408
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8247472315840156e-05,
+      "loss": 1.1625,
+      "theoretical_loss": 3.3490675318323007,
+      "tokens_seen": 3001876480
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8239447921681912e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.349056385866212,
+      "tokens_seen": 3002007552
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8231423527523672e-05,
+      "loss": 1.1776,
+      "theoretical_loss": 3.349045240523017,
+      "tokens_seen": 3002138624
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8223399133365432e-05,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3490340958026534,
+      "tokens_seen": 3002269696
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8215374739207192e-05,
+      "loss": 1.1967,
+      "theoretical_loss": 3.3490229517050585,
+      "tokens_seen": 3002400768
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.820735034504895e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.3490118082301716,
+      "tokens_seen": 3002531840
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.819932595089071e-05,
+      "loss": 1.2839,
+      "theoretical_loss": 3.34900066537793,
+      "tokens_seen": 3002662912
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8191301556732468e-05,
+      "loss": 1.2087,
+      "theoretical_loss": 3.348989523148272,
+      "tokens_seen": 3002793984
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8183277162574228e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3489783815411354,
+      "tokens_seen": 3002925056
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8175252768415985e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.348967240556458,
+      "tokens_seen": 3003056128
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8167228374257745e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.348956100194178,
+      "tokens_seen": 3003187200
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8159203980099505e-05,
+      "loss": 1.1792,
+      "theoretical_loss": 3.348944960454234,
+      "tokens_seen": 3003318272
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8151179585941264e-05,
+      "loss": 1.322,
+      "theoretical_loss": 3.348933821336564,
+      "tokens_seen": 3003449344
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.814315519178302e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3489226828411054,
+      "tokens_seen": 3003580416
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.813513079762478e-05,
+      "loss": 1.2753,
+      "theoretical_loss": 3.348911544967797,
+      "tokens_seen": 3003711488
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.812710640346654e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.348900407716576,
+      "tokens_seen": 3003842560
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8119082009308297e-05,
+      "loss": 1.2,
+      "theoretical_loss": 3.3488892710873817,
+      "tokens_seen": 3003973632
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8111057615150057e-05,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3488781350801515,
+      "tokens_seen": 3004104704
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8103033220991817e-05,
+      "loss": 1.2563,
+      "theoretical_loss": 3.3488669996948235,
+      "tokens_seen": 3004235776
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8095008826833577e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.3488558649313362,
+      "tokens_seen": 3004366848
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8086984432675334e-05,
+      "loss": 1.2866,
+      "theoretical_loss": 3.3488447307896276,
+      "tokens_seen": 3004497920
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8078960038517093e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.3488335972696355,
+      "tokens_seen": 3004628992
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8070935644358853e-05,
+      "loss": 1.2357,
+      "theoretical_loss": 3.3488224643712985,
+      "tokens_seen": 3004760064
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8062911250200613e-05,
+      "loss": 1.2689,
+      "theoretical_loss": 3.348811332094555,
+      "tokens_seen": 3004891136
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.805488685604237e-05,
+      "loss": 1.2201,
+      "theoretical_loss": 3.3488002004393422,
+      "tokens_seen": 3005022208
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.804686246188413e-05,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3487890694055995,
+      "tokens_seen": 3005153280
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.803883806772589e-05,
+      "loss": 1.1768,
+      "theoretical_loss": 3.348777938993264,
+      "tokens_seen": 3005284352
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8030813673567646e-05,
+      "loss": 1.3215,
+      "theoretical_loss": 3.3487668092022744,
+      "tokens_seen": 3005415424
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8022789279409406e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3487556800325695,
+      "tokens_seen": 3005546496
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8014764885251166e-05,
+      "loss": 1.2433,
+      "theoretical_loss": 3.348744551484087,
+      "tokens_seen": 3005677568
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8006740491092926e-05,
+      "loss": 1.2232,
+      "theoretical_loss": 3.3487334235567645,
+      "tokens_seen": 3005808640
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7998716096934682e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.348722296250541,
+      "tokens_seen": 3005939712
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7990691702776442e-05,
+      "loss": 1.1892,
+      "theoretical_loss": 3.348711169565355,
+      "tokens_seen": 3006070784
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7982667308618202e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.348700043501144,
+      "tokens_seen": 3006201856
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7974642914459962e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3486889180578467,
+      "tokens_seen": 3006332928
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.796661852030172e-05,
+      "loss": 1.3303,
+      "theoretical_loss": 3.3486777932354017,
+      "tokens_seen": 3006464000
+    },
+    {
+      "epoch": 0.82,
+      "objective/train/docs_used": 1007026,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1994062662124634,
+      "objective/train/theoretical_loss": 3.3486666690337463,
+      "objective/train/tokens_used": 1377054176,
+      "theoretical_loss": 3.3486666690337463,
+      "tokens_seen": 3006595072
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.795859412614348e-05,
+      "loss": 1.2393,
+      "theoretical_loss": 3.3486666690337463,
+      "tokens_seen": 3006595072
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.795056973198524e-05,
+      "loss": 1.2051,
+      "theoretical_loss": 3.3486555454528197,
+      "tokens_seen": 3006726144
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7942545337826995e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.3486444224925602,
+      "tokens_seen": 3006857216
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7934520943668755e-05,
+      "loss": 1.1971,
+      "theoretical_loss": 3.3486333001529056,
+      "tokens_seen": 3006988288
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7926496549510515e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3486221784337946,
+      "tokens_seen": 3007119360
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.791847215535227e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.348611057335165,
+      "tokens_seen": 3007250432
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.791044776119403e-05,
+      "loss": 1.1525,
+      "theoretical_loss": 3.3485999368569557,
+      "tokens_seen": 3007381504
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.790242336703579e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.348588816999105,
+      "tokens_seen": 3007512576
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7894398972877548e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.348577697761551,
+      "tokens_seen": 3007643648
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7886374578719307e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.348566579144232,
+      "tokens_seen": 3007774720
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7878350184561067e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.348555461147087,
+      "tokens_seen": 3007905792
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7870325790402824e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.3485443437700537,
+      "tokens_seen": 3008036864
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7862301396244584e-05,
+      "loss": 1.2061,
+      "theoretical_loss": 3.348533227013071,
+      "tokens_seen": 3008167936
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7854277002086344e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.348522110876077,
+      "tokens_seen": 3008299008
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.78462526079281e-05,
+      "loss": 1.1788,
+      "theoretical_loss": 3.34851099535901,
+      "tokens_seen": 3008430080
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.783822821376986e-05,
+      "loss": 1.2836,
+      "theoretical_loss": 3.3484998804618087,
+      "tokens_seen": 3008561152
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.783020381961162e-05,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3484887661844116,
+      "tokens_seen": 3008692224
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7822179425453377e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.3484776525267566,
+      "tokens_seen": 3008823296
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7814155031295136e-05,
+      "loss": 1.194,
+      "theoretical_loss": 3.3484665394887827,
+      "tokens_seen": 3008954368
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7806130637136896e-05,
+      "loss": 1.2755,
+      "theoretical_loss": 3.3484554270704283,
+      "tokens_seen": 3009085440
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7798106242978656e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3484443152716317,
+      "tokens_seen": 3009216512
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7790081848820413e-05,
+      "loss": 1.1795,
+      "theoretical_loss": 3.3484332040923315,
+      "tokens_seen": 3009347584
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7782057454662173e-05,
+      "loss": 1.1708,
+      "theoretical_loss": 3.3484220935324656,
+      "tokens_seen": 3009478656
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7774033060503933e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3484109835919735,
+      "tokens_seen": 3009609728
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.776600866634569e-05,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3483998742707928,
+      "tokens_seen": 3009740800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.775798427218745e-05,
+      "loss": 1.1977,
+      "theoretical_loss": 3.3483887655688624,
+      "tokens_seen": 3009871872
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.774995987802921e-05,
+      "loss": 1.2047,
+      "theoretical_loss": 3.348377657486121,
+      "tokens_seen": 3010002944
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.774193548387097e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.348366550022507,
+      "tokens_seen": 3010134016
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7733911089712725e-05,
+      "loss": 1.3116,
+      "theoretical_loss": 3.348355443177959,
+      "tokens_seen": 3010265088
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7725886695554485e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.348344336952415,
+      "tokens_seen": 3010396160
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7717862301396245e-05,
+      "loss": 1.1993,
+      "theoretical_loss": 3.3483332313458143,
+      "tokens_seen": 3010527232
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7709837907238005e-05,
+      "loss": 1.3011,
+      "theoretical_loss": 3.3483221263580947,
+      "tokens_seen": 3010658304
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.770181351307976e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.3483110219891956,
+      "tokens_seen": 3010789376
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.769378911892152e-05,
+      "loss": 1.34,
+      "theoretical_loss": 3.348299918239055,
+      "tokens_seen": 3010920448
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.768576472476328e-05,
+      "loss": 1.1856,
+      "theoretical_loss": 3.348288815107612,
+      "tokens_seen": 3011051520
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.767774033060504e-05,
+      "loss": 1.2425,
+      "theoretical_loss": 3.3482777125948044,
+      "tokens_seen": 3011182592
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7669715936446798e-05,
+      "loss": 1.2659,
+      "theoretical_loss": 3.3482666107005716,
+      "tokens_seen": 3011313664
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7661691542288558e-05,
+      "loss": 1.2005,
+      "theoretical_loss": 3.3482555094248516,
+      "tokens_seen": 3011444736
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7653667148130318e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.3482444087675836,
+      "tokens_seen": 3011575808
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7645642753972074e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.3482333087287053,
+      "tokens_seen": 3011706880
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7637618359813834e-05,
+      "loss": 1.2635,
+      "theoretical_loss": 3.348222209308157,
+      "tokens_seen": 3011837952
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7629593965655594e-05,
+      "loss": 1.2971,
+      "theoretical_loss": 3.3482111105058756,
+      "tokens_seen": 3011969024
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7621569571497354e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.3482000123218003,
+      "tokens_seen": 3012100096
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.761354517733911e-05,
+      "loss": 1.173,
+      "theoretical_loss": 3.3481889147558705,
+      "tokens_seen": 3012231168
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.760552078318087e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.348177817808024,
+      "tokens_seen": 3012362240
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.759749638902263e-05,
+      "loss": 1.1679,
+      "theoretical_loss": 3.3481667214782,
+      "tokens_seen": 3012493312
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.758947199486439e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3481556257663367,
+      "tokens_seen": 3012624384
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7581447600706147e-05,
+      "loss": 1.1905,
+      "theoretical_loss": 3.3481445306723736,
+      "tokens_seen": 3012755456
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7573423206547907e-05,
+      "loss": 1.2269,
+      "theoretical_loss": 3.3481334361962487,
+      "tokens_seen": 3012886528
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7565398812389666e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3481223423379007,
+      "tokens_seen": 3013017600
+    },
+    {
+      "epoch": 0.83,
+      "objective/train/docs_used": 1009292,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1828504800796509,
+      "objective/train/theoretical_loss": 3.348111249097269,
+      "objective/train/tokens_used": 1383607776,
+      "theoretical_loss": 3.348111249097269,
+      "tokens_seen": 3013148672
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7557374418231423e-05,
+      "loss": 1.1701,
+      "theoretical_loss": 3.348111249097269,
+      "tokens_seen": 3013148672
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7549350024073183e-05,
+      "loss": 1.2118,
+      "theoretical_loss": 3.348100156474292,
+      "tokens_seen": 3013279744
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7541325629914943e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.3480890644689074,
+      "tokens_seen": 3013410816
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7533301235756703e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3480779730810557,
+      "tokens_seen": 3013541888
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.752527684159846e-05,
+      "loss": 1.2491,
+      "theoretical_loss": 3.3480668823106745,
+      "tokens_seen": 3013672960
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.751725244744022e-05,
+      "loss": 1.2663,
+      "theoretical_loss": 3.348055792157703,
+      "tokens_seen": 3013804032
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.750922805328198e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.3480447026220803,
+      "tokens_seen": 3013935104
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.750120365912374e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3480336137037443,
+      "tokens_seen": 3014066176
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7493179264965495e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.3480225254026346,
+      "tokens_seen": 3014197248
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7485154870807255e-05,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3480114377186894,
+      "tokens_seen": 3014328320
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7477130476649015e-05,
+      "loss": 1.2288,
+      "theoretical_loss": 3.348000350651848,
+      "tokens_seen": 3014459392
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7469106082490772e-05,
+      "loss": 1.2133,
+      "theoretical_loss": 3.3479892642020492,
+      "tokens_seen": 3014590464
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.746108168833253e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.3479781783692317,
+      "tokens_seen": 3014721536
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.745305729417429e-05,
+      "loss": 1.2338,
+      "theoretical_loss": 3.347967093153334,
+      "tokens_seen": 3014852608
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.744503290001605e-05,
+      "loss": 1.2095,
+      "theoretical_loss": 3.3479560085542954,
+      "tokens_seen": 3014983680
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7437008505857808e-05,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3479449245720545,
+      "tokens_seen": 3015114752
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7428984111699568e-05,
+      "loss": 1.2755,
+      "theoretical_loss": 3.3479338412065505,
+      "tokens_seen": 3015245824
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7420959717541328e-05,
+      "loss": 1.2334,
+      "theoretical_loss": 3.347922758457722,
+      "tokens_seen": 3015376896
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7412935323383088e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.347911676325508,
+      "tokens_seen": 3015507968
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7404910929224844e-05,
+      "loss": 1.1945,
+      "theoretical_loss": 3.3479005948098473,
+      "tokens_seen": 3015639040
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7396886535066604e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.347889513910679,
+      "tokens_seen": 3015770112
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7388862140908364e-05,
+      "loss": 1.3421,
+      "theoretical_loss": 3.3478784336279417,
+      "tokens_seen": 3015901184
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.738083774675012e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.3478673539615746,
+      "tokens_seen": 3016032256
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.737281335259188e-05,
+      "loss": 1.2168,
+      "theoretical_loss": 3.3478562749115164,
+      "tokens_seen": 3016163328
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.736478895843364e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.347845196477706,
+      "tokens_seen": 3016294400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.73567645642754e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.347834118660083,
+      "tokens_seen": 3016425472
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7348740170117157e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.3478230414585854,
+      "tokens_seen": 3016556544
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7340715775958917e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3478119648731526,
+      "tokens_seen": 3016687616
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7332691381800677e-05,
+      "loss": 1.1253,
+      "theoretical_loss": 3.347800888903724,
+      "tokens_seen": 3016818688
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7324666987642436e-05,
+      "loss": 1.2487,
+      "theoretical_loss": 3.3477898135502375,
+      "tokens_seen": 3016949760
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7316642593484193e-05,
+      "loss": 1.2231,
+      "theoretical_loss": 3.347778738812633,
+      "tokens_seen": 3017080832
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7308618199325953e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.3477676646908496,
+      "tokens_seen": 3017211904
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7300593805167713e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.347756591184826,
+      "tokens_seen": 3017342976
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.729256941100947e-05,
+      "loss": 1.2455,
+      "theoretical_loss": 3.347745518294501,
+      "tokens_seen": 3017474048
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.728454501685123e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3477344460198135,
+      "tokens_seen": 3017605120
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.727652062269299e-05,
+      "loss": 1.2969,
+      "theoretical_loss": 3.347723374360703,
+      "tokens_seen": 3017736192
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7268496228534746e-05,
+      "loss": 1.2223,
+      "theoretical_loss": 3.347712303317109,
+      "tokens_seen": 3017867264
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7260471834376506e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.347701232888969,
+      "tokens_seen": 3017998336
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7252447440218265e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.3476901630762232,
+      "tokens_seen": 3018129408
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7244423046060022e-05,
+      "loss": 1.2472,
+      "theoretical_loss": 3.347679093878811,
+      "tokens_seen": 3018260480
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7236398651901782e-05,
+      "loss": 1.2209,
+      "theoretical_loss": 3.34766802529667,
+      "tokens_seen": 3018391552
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7228374257743542e-05,
+      "loss": 1.3294,
+      "theoretical_loss": 3.347656957329741,
+      "tokens_seen": 3018522624
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.72203498635853e-05,
+      "loss": 1.2963,
+      "theoretical_loss": 3.347645889977962,
+      "tokens_seen": 3018653696
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7212325469427058e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3476348232412723,
+      "tokens_seen": 3018784768
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7204301075268818e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.3476237571196115,
+      "tokens_seen": 3018915840
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7196276681110575e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.347612691612918,
+      "tokens_seen": 3019046912
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7188252286952335e-05,
+      "loss": 1.2569,
+      "theoretical_loss": 3.3476016267211315,
+      "tokens_seen": 3019177984
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7180227892794094e-05,
+      "loss": 1.1943,
+      "theoretical_loss": 3.3475905624441906,
+      "tokens_seen": 3019309056
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.717220349863585e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.347579498782035,
+      "tokens_seen": 3019440128
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.716417910447761e-05,
+      "loss": 1.2955,
+      "theoretical_loss": 3.347568435734604,
+      "tokens_seen": 3019571200
+    },
+    {
+      "epoch": 0.83,
+      "objective/train/docs_used": 1011539,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1783403158187866,
+      "objective/train/theoretical_loss": 3.3475573733018353,
+      "objective/train/tokens_used": 1390161376,
+      "theoretical_loss": 3.3475573733018353,
+      "tokens_seen": 3019702272
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.715615471031937e-05,
+      "loss": 1.2063,
+      "theoretical_loss": 3.3475573733018353,
+      "tokens_seen": 3019702272
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.714813031616113e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.34754631148367,
+      "tokens_seen": 3019833344
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7140105922002887e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.347535250280046,
+      "tokens_seen": 3019964416
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7132081527844647e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.347524189690903,
+      "tokens_seen": 3020095488
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7124057133686407e-05,
+      "loss": 1.1899,
+      "theoretical_loss": 3.34751312971618,
+      "tokens_seen": 3020226560
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7116032739528167e-05,
+      "loss": 1.2466,
+      "theoretical_loss": 3.3475020703558167,
+      "tokens_seen": 3020357632
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7108008345369923e-05,
+      "loss": 1.2204,
+      "theoretical_loss": 3.3474910116097516,
+      "tokens_seen": 3020488704
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7099983951211683e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3474799534779245,
+      "tokens_seen": 3020619776
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7091959557053443e-05,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3474688959602745,
+      "tokens_seen": 3020750848
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.70839351628952e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3474578390567404,
+      "tokens_seen": 3020881920
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.707591076873696e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.347446782767262,
+      "tokens_seen": 3021012992
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.706788637457872e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.347435727091778,
+      "tokens_seen": 3021144064
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.705986198042048e-05,
+      "loss": 1.1553,
+      "theoretical_loss": 3.347424672030228,
+      "tokens_seen": 3021275136
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7051837586262236e-05,
+      "loss": 1.1763,
+      "theoretical_loss": 3.3474136175825517,
+      "tokens_seen": 3021406208
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7043813192103996e-05,
+      "loss": 1.1915,
+      "theoretical_loss": 3.3474025637486875,
+      "tokens_seen": 3021537280
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7035788797945756e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.3473915105285754,
+      "tokens_seen": 3021668352
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7027764403787516e-05,
+      "loss": 1.2914,
+      "theoretical_loss": 3.3473804579221547,
+      "tokens_seen": 3021799424
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7019740009629272e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.347369405929364,
+      "tokens_seen": 3021930496
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7011715615471032e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.3473583545501433,
+      "tokens_seen": 3022061568
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7003691221312792e-05,
+      "loss": 1.3125,
+      "theoretical_loss": 3.3473473037844315,
+      "tokens_seen": 3022192640
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.699566682715455e-05,
+      "loss": 1.2347,
+      "theoretical_loss": 3.3473362536321685,
+      "tokens_seen": 3022323712
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.698764243299631e-05,
+      "loss": 1.3273,
+      "theoretical_loss": 3.347325204093293,
+      "tokens_seen": 3022454784
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.697961803883807e-05,
+      "loss": 1.2177,
+      "theoretical_loss": 3.3473141551677448,
+      "tokens_seen": 3022585856
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.697159364467983e-05,
+      "loss": 1.2596,
+      "theoretical_loss": 3.3473031068554633,
+      "tokens_seen": 3022716928
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6963569250521585e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3472920591563873,
+      "tokens_seen": 3022848000
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6955544856363345e-05,
+      "loss": 1.1734,
+      "theoretical_loss": 3.3472810120704564,
+      "tokens_seen": 3022979072
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6947520462205105e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3472699655976106,
+      "tokens_seen": 3023110144
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6939496068046865e-05,
+      "loss": 1.2512,
+      "theoretical_loss": 3.3472589197377887,
+      "tokens_seen": 3023241216
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.693147167388862e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.34724787449093,
+      "tokens_seen": 3023372288
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.692344727973038e-05,
+      "loss": 1.1889,
+      "theoretical_loss": 3.3472368298569743,
+      "tokens_seen": 3023503360
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.691542288557214e-05,
+      "loss": 1.29,
+      "theoretical_loss": 3.347225785835861,
+      "tokens_seen": 3023634432
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6907398491413897e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3472147424275294,
+      "tokens_seen": 3023765504
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6899374097255657e-05,
+      "loss": 1.2395,
+      "theoretical_loss": 3.347203699631919,
+      "tokens_seen": 3023896576
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6891349703097417e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.347192657448969,
+      "tokens_seen": 3024027648
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6883325308939177e-05,
+      "loss": 1.3337,
+      "theoretical_loss": 3.347181615878619,
+      "tokens_seen": 3024158720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6875300914780934e-05,
+      "loss": 1.2207,
+      "theoretical_loss": 3.347170574920809,
+      "tokens_seen": 3024289792
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6867276520622694e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.3471595345754777,
+      "tokens_seen": 3024420864
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6859252126464453e-05,
+      "loss": 1.2734,
+      "theoretical_loss": 3.3471484948425645,
+      "tokens_seen": 3024551936
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6851227732306213e-05,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3471374557220095,
+      "tokens_seen": 3024683008
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.684320333814797e-05,
+      "loss": 1.276,
+      "theoretical_loss": 3.3471264172137523,
+      "tokens_seen": 3024814080
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.683517894398973e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.347115379317732,
+      "tokens_seen": 3024945152
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.682715454983149e-05,
+      "loss": 1.2869,
+      "theoretical_loss": 3.3471043420338877,
+      "tokens_seen": 3025076224
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6819130155673246e-05,
+      "loss": 1.1933,
+      "theoretical_loss": 3.3470933053621597,
+      "tokens_seen": 3025207296
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6811105761515006e-05,
+      "loss": 1.2873,
+      "theoretical_loss": 3.3470822693024873,
+      "tokens_seen": 3025338368
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6803081367356766e-05,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3470712338548103,
+      "tokens_seen": 3025469440
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6795056973198526e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3470601990190674,
+      "tokens_seen": 3025600512
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6787032579040282e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.347049164795199,
+      "tokens_seen": 3025731584
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6779008184882042e-05,
+      "loss": 1.1725,
+      "theoretical_loss": 3.3470381311831443,
+      "tokens_seen": 3025862656
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6770983790723802e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.347027098182843,
+      "tokens_seen": 3025993728
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6762959396565562e-05,
+      "loss": 1.1865,
+      "theoretical_loss": 3.3470160657942345,
+      "tokens_seen": 3026124800
+    },
+    {
+      "epoch": 0.83,
+      "objective/train/docs_used": 1013697,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3000990152359009,
+      "objective/train/theoretical_loss": 3.3470050340172586,
+      "objective/train/tokens_used": 1396714976,
+      "theoretical_loss": 3.3470050340172586,
+      "tokens_seen": 3026255872
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.675493500240732e-05,
+      "loss": 1.2792,
+      "theoretical_loss": 3.3470050340172586,
+      "tokens_seen": 3026255872
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.674691060824908e-05,
+      "loss": 1.2181,
+      "theoretical_loss": 3.3469940028518548,
+      "tokens_seen": 3026386944
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.673888621409084e-05,
+      "loss": 1.1923,
+      "theoretical_loss": 3.3469829722979627,
+      "tokens_seen": 3026518016
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.67308618199326e-05,
+      "loss": 1.2666,
+      "theoretical_loss": 3.346971942355522,
+      "tokens_seen": 3026649088
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6722837425774355e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.346960913024472,
+      "tokens_seen": 3026780160
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6714813031616115e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.3469498843047534,
+      "tokens_seen": 3026911232
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6706788637457875e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3469388561963047,
+      "tokens_seen": 3027042304
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.669876424329963e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.346927828699066,
+      "tokens_seen": 3027173376
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.669073984914139e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.3469168018129762,
+      "tokens_seen": 3027304448
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.668271545498315e-05,
+      "loss": 1.2145,
+      "theoretical_loss": 3.3469057755379765,
+      "tokens_seen": 3027435520
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.667469106082491e-05,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3468947498740054,
+      "tokens_seen": 3027566592
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.346883724821003,
+      "tokens_seen": 3027697664
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6658642272508427e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.346872700378909,
+      "tokens_seen": 3027828736
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6650617878350187e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.346861676547663,
+      "tokens_seen": 3027959808
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6642593484191947e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.3468506533272047,
+      "tokens_seen": 3028090880
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6634569090033704e-05,
+      "loss": 1.1979,
+      "theoretical_loss": 3.3468396307174735,
+      "tokens_seen": 3028221952
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6626544695875464e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.34682860871841,
+      "tokens_seen": 3028353024
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6618520301717224e-05,
+      "loss": 1.285,
+      "theoretical_loss": 3.3468175873299533,
+      "tokens_seen": 3028484096
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.661049590755898e-05,
+      "loss": 1.2075,
+      "theoretical_loss": 3.346806566552043,
+      "tokens_seen": 3028615168
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.660247151340074e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.3467955463846195,
+      "tokens_seen": 3028746240
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.65944471192425e-05,
+      "loss": 1.2759,
+      "theoretical_loss": 3.346784526827622,
+      "tokens_seen": 3028877312
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6586422725084256e-05,
+      "loss": 1.2371,
+      "theoretical_loss": 3.346773507880991,
+      "tokens_seen": 3029008384
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6578398330926016e-05,
+      "loss": 1.3045,
+      "theoretical_loss": 3.3467624895446653,
+      "tokens_seen": 3029139456
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6570373936767776e-05,
+      "loss": 1.2091,
+      "theoretical_loss": 3.346751471818585,
+      "tokens_seen": 3029270528
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6562349542609533e-05,
+      "loss": 1.251,
+      "theoretical_loss": 3.34674045470269,
+      "tokens_seen": 3029401600
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6554325148451293e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.346729438196921,
+      "tokens_seen": 3029532672
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6546300754293053e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3467184223012163,
+      "tokens_seen": 3029663744
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.653827636013481e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.3467074070155163,
+      "tokens_seen": 3029794816
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.653025196597657e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.3466963923397612,
+      "tokens_seen": 3029925888
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.652222757181833e-05,
+      "loss": 1.2723,
+      "theoretical_loss": 3.3466853782738903,
+      "tokens_seen": 3030056960
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6514203177660085e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3466743648178445,
+      "tokens_seen": 3030188032
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6506178783501845e-05,
+      "loss": 1.171,
+      "theoretical_loss": 3.346663351971562,
+      "tokens_seen": 3030319104
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6498154389343605e-05,
+      "loss": 1.2801,
+      "theoretical_loss": 3.346652339734984,
+      "tokens_seen": 3030450176
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.649012999518536e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.34664132810805,
+      "tokens_seen": 3030581248
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.648210560102712e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3466303170906997,
+      "tokens_seen": 3030712320
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.647408120686888e-05,
+      "loss": 1.2733,
+      "theoretical_loss": 3.346619306682873,
+      "tokens_seen": 3030843392
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.646605681271064e-05,
+      "loss": 1.2954,
+      "theoretical_loss": 3.34660829688451,
+      "tokens_seen": 3030974464
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6458032418552398e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.3465972876955505,
+      "tokens_seen": 3031105536
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6450008024394158e-05,
+      "loss": 1.2711,
+      "theoretical_loss": 3.3465862791159346,
+      "tokens_seen": 3031236608
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6441983630235918e-05,
+      "loss": 1.2059,
+      "theoretical_loss": 3.346575271145602,
+      "tokens_seen": 3031367680
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6433959236077674e-05,
+      "loss": 1.2857,
+      "theoretical_loss": 3.346564263784493,
+      "tokens_seen": 3031498752
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6425934841919434e-05,
+      "loss": 1.2927,
+      "theoretical_loss": 3.3465532570325465,
+      "tokens_seen": 3031629824
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6417910447761194e-05,
+      "loss": 1.253,
+      "theoretical_loss": 3.346542250889704,
+      "tokens_seen": 3031760896
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6409886053602954e-05,
+      "loss": 1.2728,
+      "theoretical_loss": 3.346531245355904,
+      "tokens_seen": 3031891968
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.640186165944471e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3465202404310874,
+      "tokens_seen": 3032023040
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.639383726528647e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.3465092361151942,
+      "tokens_seen": 3032154112
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.638581287112823e-05,
+      "loss": 1.1778,
+      "theoretical_loss": 3.346498232408164,
+      "tokens_seen": 3032285184
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.637778847696999e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.3464872293099366,
+      "tokens_seen": 3032416256
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6369764082811747e-05,
+      "loss": 1.3082,
+      "theoretical_loss": 3.3464762268204526,
+      "tokens_seen": 3032547328
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6361739688653507e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3464652249396516,
+      "tokens_seen": 3032678400
+    },
+    {
+      "epoch": 0.84,
+      "objective/train/docs_used": 1015863,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2626835107803345,
+      "objective/train/theoretical_loss": 3.3464542236674735,
+      "objective/train/tokens_used": 1403268576,
+      "theoretical_loss": 3.3464542236674735,
+      "tokens_seen": 3032809472
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6353715294495267e-05,
+      "loss": 1.2308,
+      "theoretical_loss": 3.3464542236674735,
+      "tokens_seen": 3032809472
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6345690900337023e-05,
+      "loss": 1.3119,
+      "theoretical_loss": 3.346443223003859,
+      "tokens_seen": 3032940544
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6337666506178783e-05,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3464322229487475,
+      "tokens_seen": 3033071616
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6329642112020543e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.3464212235020794,
+      "tokens_seen": 3033202688
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6321617717862303e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.3464102246637943,
+      "tokens_seen": 3033333760
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.631359332370406e-05,
+      "loss": 1.2281,
+      "theoretical_loss": 3.346399226433833,
+      "tokens_seen": 3033464832
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.630556892954582e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3463882288121347,
+      "tokens_seen": 3033595904
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.629754453538758e-05,
+      "loss": 1.2164,
+      "theoretical_loss": 3.34637723179864,
+      "tokens_seen": 3033726976
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.628952014122934e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3463662353932895,
+      "tokens_seen": 3033858048
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6281495747071096e-05,
+      "loss": 1.2821,
+      "theoretical_loss": 3.346355239596022,
+      "tokens_seen": 3033989120
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6273471352912855e-05,
+      "loss": 1.1495,
+      "theoretical_loss": 3.3463442444067786,
+      "tokens_seen": 3034120192
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6265446958754615e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3463332498254994,
+      "tokens_seen": 3034251264
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6257422564596375e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3463222558521237,
+      "tokens_seen": 3034382336
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6249398170438132e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.3463112624865925,
+      "tokens_seen": 3034513408
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.624137377627989e-05,
+      "loss": 1.2551,
+      "theoretical_loss": 3.3463002697288458,
+      "tokens_seen": 3034644480
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.623334938212165e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.3462892775788236,
+      "tokens_seen": 3034775552
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6225324987963408e-05,
+      "loss": 1.2593,
+      "theoretical_loss": 3.3462782860364655,
+      "tokens_seen": 3034906624
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6217300593805168e-05,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3462672951017125,
+      "tokens_seen": 3035037696
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6209276199646928e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.3462563047745046,
+      "tokens_seen": 3035168768
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6201251805488688e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.346245315054782,
+      "tokens_seen": 3035299840
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6193227411330444e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.3462343259424845,
+      "tokens_seen": 3035430912
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6185203017172204e-05,
+      "loss": 1.2844,
+      "theoretical_loss": 3.3462233374375523,
+      "tokens_seen": 3035561984
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6177178623013964e-05,
+      "loss": 1.2898,
+      "theoretical_loss": 3.3462123495399263,
+      "tokens_seen": 3035693056
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6169154228855724e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.346201362249546,
+      "tokens_seen": 3035824128
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.616112983469748e-05,
+      "loss": 1.1825,
+      "theoretical_loss": 3.3461903755663522,
+      "tokens_seen": 3035955200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.615310544053924e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.3461793894902843,
+      "tokens_seen": 3036086272
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6145081046381e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.3461684040212836,
+      "tokens_seen": 3036217344
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6137056652222757e-05,
+      "loss": 1.2931,
+      "theoretical_loss": 3.3461574191592893,
+      "tokens_seen": 3036348416
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6129032258064517e-05,
+      "loss": 1.2785,
+      "theoretical_loss": 3.3461464349042425,
+      "tokens_seen": 3036479488
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6121007863906277e-05,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3461354512560835,
+      "tokens_seen": 3036610560
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6112983469748037e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3461244682147515,
+      "tokens_seen": 3036741632
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6104959075589793e-05,
+      "loss": 1.2884,
+      "theoretical_loss": 3.346113485780188,
+      "tokens_seen": 3036872704
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6096934681431553e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3461025039523324,
+      "tokens_seen": 3037003776
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6088910287273313e-05,
+      "loss": 1.2656,
+      "theoretical_loss": 3.346091522731126,
+      "tokens_seen": 3037134848
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6080885893115073e-05,
+      "loss": 1.2449,
+      "theoretical_loss": 3.346080542116508,
+      "tokens_seen": 3037265920
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.607286149895683e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.3460695621084193,
+      "tokens_seen": 3037396992
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.606483710479859e-05,
+      "loss": 1.1735,
+      "theoretical_loss": 3.3460585827068003,
+      "tokens_seen": 3037528064
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.605681271064035e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3460476039115914,
+      "tokens_seen": 3037659136
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6048788316482106e-05,
+      "loss": 1.2677,
+      "theoretical_loss": 3.346036625722732,
+      "tokens_seen": 3037790208
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6040763922323866e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.346025648140164,
+      "tokens_seen": 3037921280
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6032739528165626e-05,
+      "loss": 1.2332,
+      "theoretical_loss": 3.3460146711638266,
+      "tokens_seen": 3038052352
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6024715134007385e-05,
+      "loss": 1.24,
+      "theoretical_loss": 3.3460036947936604,
+      "tokens_seen": 3038183424
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6016690739849142e-05,
+      "loss": 1.3228,
+      "theoretical_loss": 3.345992719029606,
+      "tokens_seen": 3038314496
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6008666345690902e-05,
+      "loss": 1.3071,
+      "theoretical_loss": 3.3459817438716035,
+      "tokens_seen": 3038445568
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.6000641951532662e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3459707693195937,
+      "tokens_seen": 3038576640
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.599261755737442e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.345959795373517,
+      "tokens_seen": 3038707712
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5984593163216178e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.3459488220333133,
+      "tokens_seen": 3038838784
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5976568769057938e-05,
+      "loss": 1.1897,
+      "theoretical_loss": 3.3459378492989234,
+      "tokens_seen": 3038969856
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5968544374899698e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.345926877170287,
+      "tokens_seen": 3039100928
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5960519980741454e-05,
+      "loss": 1.2669,
+      "theoretical_loss": 3.345915905647346,
+      "tokens_seen": 3039232000
+    },
+    {
+      "epoch": 0.84,
+      "objective/train/docs_used": 1018122,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2888078689575195,
+      "objective/train/theoretical_loss": 3.3459049347300396,
+      "objective/train/tokens_used": 1409822176,
+      "theoretical_loss": 3.3459049347300396,
+      "tokens_seen": 3039363072
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5952495586583214e-05,
+      "loss": 1.2698,
+      "theoretical_loss": 3.3459049347300396,
+      "tokens_seen": 3039363072
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5944471192424974e-05,
+      "loss": 1.2646,
+      "theoretical_loss": 3.345893964418309,
+      "tokens_seen": 3039494144
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5936446798266734e-05,
+      "loss": 1.193,
+      "theoretical_loss": 3.345882994712094,
+      "tokens_seen": 3039625216
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.592842240410849e-05,
+      "loss": 1.2667,
+      "theoretical_loss": 3.3458720256113352,
+      "tokens_seen": 3039756288
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.592039800995025e-05,
+      "loss": 1.2617,
+      "theoretical_loss": 3.345861057115974,
+      "tokens_seen": 3039887360
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.591237361579201e-05,
+      "loss": 1.2084,
+      "theoretical_loss": 3.3458500892259493,
+      "tokens_seen": 3040018432
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5904349221633767e-05,
+      "loss": 1.3319,
+      "theoretical_loss": 3.345839121941203,
+      "tokens_seen": 3040149504
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5896324827475527e-05,
+      "loss": 1.2082,
+      "theoretical_loss": 3.345828155261675,
+      "tokens_seen": 3040280576
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5888300433317287e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3458171891873056,
+      "tokens_seen": 3040411648
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5880276039159043e-05,
+      "loss": 1.2346,
+      "theoretical_loss": 3.3458062237180357,
+      "tokens_seen": 3040542720
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5872251645000803e-05,
+      "loss": 1.2788,
+      "theoretical_loss": 3.345795258853806,
+      "tokens_seen": 3040673792
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5864227250842563e-05,
+      "loss": 1.2892,
+      "theoretical_loss": 3.3457842945945564,
+      "tokens_seen": 3040804864
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.585620285668432e-05,
+      "loss": 1.2338,
+      "theoretical_loss": 3.345773330940228,
+      "tokens_seen": 3040935936
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.584817846252608e-05,
+      "loss": 1.2019,
+      "theoretical_loss": 3.345762367890761,
+      "tokens_seen": 3041067008
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.584015406836784e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.345751405446096,
+      "tokens_seen": 3041198080
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5832129674209596e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.345740443606174,
+      "tokens_seen": 3041329152
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5824105280051356e-05,
+      "loss": 1.2826,
+      "theoretical_loss": 3.345729482370935,
+      "tokens_seen": 3041460224
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5816080885893116e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.34571852174032,
+      "tokens_seen": 3041591296
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5808056491734872e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.3457075617142698,
+      "tokens_seen": 3041722368
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5800032097576632e-05,
+      "loss": 1.3367,
+      "theoretical_loss": 3.3456966022927244,
+      "tokens_seen": 3041853440
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5792007703418392e-05,
+      "loss": 1.2989,
+      "theoretical_loss": 3.3456856434756244,
+      "tokens_seen": 3041984512
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.578398330926015e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.345674685262911,
+      "tokens_seen": 3042115584
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.577595891510191e-05,
+      "loss": 1.273,
+      "theoretical_loss": 3.3456637276545242,
+      "tokens_seen": 3042246656
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.576793452094367e-05,
+      "loss": 1.2553,
+      "theoretical_loss": 3.345652770650405,
+      "tokens_seen": 3042377728
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.575991012678543e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.3456418142504942,
+      "tokens_seen": 3042508800
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5751885732627185e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.345630858454732,
+      "tokens_seen": 3042639872
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5743861338468945e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3456199032630596,
+      "tokens_seen": 3042770944
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5735836944310705e-05,
+      "loss": 1.2166,
+      "theoretical_loss": 3.3456089486754172,
+      "tokens_seen": 3042902016
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5727812550152465e-05,
+      "loss": 1.1851,
+      "theoretical_loss": 3.345597994691746,
+      "tokens_seen": 3043033088
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.571978815599422e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.345587041311986,
+      "tokens_seen": 3043164160
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.571176376183598e-05,
+      "loss": 1.2192,
+      "theoretical_loss": 3.3455760885360784,
+      "tokens_seen": 3043295232
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.570373936767774e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.3455651363639634,
+      "tokens_seen": 3043426304
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.56957149735195e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3455541847955823,
+      "tokens_seen": 3043557376
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5687690579361257e-05,
+      "loss": 1.182,
+      "theoretical_loss": 3.3455432338308757,
+      "tokens_seen": 3043688448
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5679666185203017e-05,
+      "loss": 1.2539,
+      "theoretical_loss": 3.345532283469784,
+      "tokens_seen": 3043819520
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5671641791044777e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3455213337122487,
+      "tokens_seen": 3043950592
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5663617396886534e-05,
+      "loss": 1.1994,
+      "theoretical_loss": 3.3455103845582093,
+      "tokens_seen": 3044081664
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5655593002728294e-05,
+      "loss": 1.2225,
+      "theoretical_loss": 3.345499436007608,
+      "tokens_seen": 3044212736
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5647568608570054e-05,
+      "loss": 1.2531,
+      "theoretical_loss": 3.345488488060384,
+      "tokens_seen": 3044343808
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5639544214411813e-05,
+      "loss": 1.2379,
+      "theoretical_loss": 3.3454775407164794,
+      "tokens_seen": 3044474880
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.563151982025357e-05,
+      "loss": 1.2248,
+      "theoretical_loss": 3.345466593975835,
+      "tokens_seen": 3044605952
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.562349542609533e-05,
+      "loss": 1.261,
+      "theoretical_loss": 3.34545564783839,
+      "tokens_seen": 3044737024
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.561547103193709e-05,
+      "loss": 1.195,
+      "theoretical_loss": 3.3454447023040865,
+      "tokens_seen": 3044868096
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.560744663777885e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.3454337573728656,
+      "tokens_seen": 3044999168
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5599422243620606e-05,
+      "loss": 1.1869,
+      "theoretical_loss": 3.3454228130446673,
+      "tokens_seen": 3045130240
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5591397849462366e-05,
+      "loss": 1.3238,
+      "theoretical_loss": 3.3454118693194324,
+      "tokens_seen": 3045261312
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5583373455304126e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.3454009261971023,
+      "tokens_seen": 3045392384
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5575349061145883e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3453899836776175,
+      "tokens_seen": 3045523456
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5567324666987642e-05,
+      "loss": 1.2399,
+      "theoretical_loss": 3.345379041760919,
+      "tokens_seen": 3045654528
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5559300272829402e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.3453681004469473,
+      "tokens_seen": 3045785600
+    },
+    {
+      "epoch": 0.85,
+      "objective/train/docs_used": 1020383,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1158528327941895,
+      "objective/train/theoretical_loss": 3.345357159735644,
+      "objective/train/tokens_used": 1416375776,
+      "theoretical_loss": 3.345357159735644,
+      "tokens_seen": 3045916672
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5551275878671162e-05,
+      "loss": 1.1367,
+      "theoretical_loss": 3.345357159735644,
+      "tokens_seen": 3045916672
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.554325148451292e-05,
+      "loss": 1.2778,
+      "theoretical_loss": 3.3453462196269497,
+      "tokens_seen": 3046047744
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.553522709035468e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.3453352801208043,
+      "tokens_seen": 3046178816
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.552720269619644e-05,
+      "loss": 1.1799,
+      "theoretical_loss": 3.3453243412171503,
+      "tokens_seen": 3046309888
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.55191783020382e-05,
+      "loss": 1.3025,
+      "theoretical_loss": 3.3453134029159273,
+      "tokens_seen": 3046440960
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5511153907879955e-05,
+      "loss": 1.1893,
+      "theoretical_loss": 3.3453024652170766,
+      "tokens_seen": 3046572032
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5503129513721715e-05,
+      "loss": 1.2382,
+      "theoretical_loss": 3.345291528120539,
+      "tokens_seen": 3046703104
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5495105119563475e-05,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3452805916262562,
+      "tokens_seen": 3046834176
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.548708072540523e-05,
+      "loss": 1.169,
+      "theoretical_loss": 3.345269655734168,
+      "tokens_seen": 3046965248
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.547905633124699e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.345258720444216,
+      "tokens_seen": 3047096320
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.547103193708875e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3452477857563414,
+      "tokens_seen": 3047227392
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.546300754293051e-05,
+      "loss": 1.1414,
+      "theoretical_loss": 3.3452368516704847,
+      "tokens_seen": 3047358464
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5454983148772268e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.345225918186587,
+      "tokens_seen": 3047489536
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5446958754614027e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.345214985304589,
+      "tokens_seen": 3047620608
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5438934360455787e-05,
+      "loss": 1.3265,
+      "theoretical_loss": 3.345204053024432,
+      "tokens_seen": 3047751680
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5430909966297547e-05,
+      "loss": 1.2144,
+      "theoretical_loss": 3.345193121346057,
+      "tokens_seen": 3047882752
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5422885572139304e-05,
+      "loss": 1.2784,
+      "theoretical_loss": 3.345182190269404,
+      "tokens_seen": 3048013824
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5414861177981064e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.345171259794416,
+      "tokens_seen": 3048144896
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5406836783822824e-05,
+      "loss": 1.214,
+      "theoretical_loss": 3.3451603299210326,
+      "tokens_seen": 3048275968
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.539881238966458e-05,
+      "loss": 1.2964,
+      "theoretical_loss": 3.345149400649195,
+      "tokens_seen": 3048407040
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.539078799550634e-05,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3451384719788444,
+      "tokens_seen": 3048538112
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.53827636013481e-05,
+      "loss": 1.237,
+      "theoretical_loss": 3.345127543909922,
+      "tokens_seen": 3048669184
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.537473920718986e-05,
+      "loss": 1.2908,
+      "theoretical_loss": 3.3451166164423682,
+      "tokens_seen": 3048800256
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5366714813031616e-05,
+      "loss": 1.2278,
+      "theoretical_loss": 3.345105689576125,
+      "tokens_seen": 3048931328
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5358690418873376e-05,
+      "loss": 1.2695,
+      "theoretical_loss": 3.345094763311133,
+      "tokens_seen": 3049062400
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5350666024715136e-05,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3450838376473326,
+      "tokens_seen": 3049193472
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5342641630556896e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.345072912584666,
+      "tokens_seen": 3049324544
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5334617236398653e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.345061988123074,
+      "tokens_seen": 3049455616
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5326592842240413e-05,
+      "loss": 1.1947,
+      "theoretical_loss": 3.345051064262497,
+      "tokens_seen": 3049586688
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5318568448082172e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.345040141002877,
+      "tokens_seen": 3049717760
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.531054405392393e-05,
+      "loss": 1.2997,
+      "theoretical_loss": 3.3450292183441546,
+      "tokens_seen": 3049848832
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.530251965976569e-05,
+      "loss": 1.2535,
+      "theoretical_loss": 3.3450182962862707,
+      "tokens_seen": 3049979904
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.529449526560745e-05,
+      "loss": 1.2175,
+      "theoretical_loss": 3.345007374829167,
+      "tokens_seen": 3050110976
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.528647087144921e-05,
+      "loss": 1.2937,
+      "theoretical_loss": 3.3449964539727848,
+      "tokens_seen": 3050242048
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5278446477290965e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.3449855337170646,
+      "tokens_seen": 3050373120
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5270422083132725e-05,
+      "loss": 1.2213,
+      "theoretical_loss": 3.3449746140619476,
+      "tokens_seen": 3050504192
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5262397688974485e-05,
+      "loss": 1.3118,
+      "theoretical_loss": 3.344963695007375,
+      "tokens_seen": 3050635264
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5254373294816243e-05,
+      "loss": 1.3078,
+      "theoretical_loss": 3.3449527765532885,
+      "tokens_seen": 3050766336
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5246348900658e-05,
+      "loss": 1.2831,
+      "theoretical_loss": 3.344941858699629,
+      "tokens_seen": 3050897408
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.523832450649976e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.3449309414463375,
+      "tokens_seen": 3051028480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.523030011234152e-05,
+      "loss": 1.1757,
+      "theoretical_loss": 3.3449200247933555,
+      "tokens_seen": 3051159552
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.522227571818328e-05,
+      "loss": 1.2337,
+      "theoretical_loss": 3.344909108740624,
+      "tokens_seen": 3051290624
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5214251324025036e-05,
+      "loss": 1.2147,
+      "theoretical_loss": 3.344898193288084,
+      "tokens_seen": 3051421696
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5206226929866796e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.344887278435677,
+      "tokens_seen": 3051552768
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5198202535708556e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.344876364183344,
+      "tokens_seen": 3051683840
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5190178141550312e-05,
+      "loss": 1.3356,
+      "theoretical_loss": 3.3448654505310267,
+      "tokens_seen": 3051814912
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5182153747392072e-05,
+      "loss": 1.2849,
+      "theoretical_loss": 3.344854537478666,
+      "tokens_seen": 3051945984
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5174129353233832e-05,
+      "loss": 1.2701,
+      "theoretical_loss": 3.3448436250262032,
+      "tokens_seen": 3052077056
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5166104959075592e-05,
+      "loss": 1.2483,
+      "theoretical_loss": 3.3448327131735796,
+      "tokens_seen": 3052208128
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5158080564917349e-05,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3448218019207365,
+      "tokens_seen": 3052339200
+    },
+    {
+      "epoch": 0.85,
+      "objective/train/docs_used": 1022554,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.289522409439087,
+      "objective/train/theoretical_loss": 3.344810891267615,
+      "objective/train/tokens_used": 1422929376,
+      "theoretical_loss": 3.344810891267615,
+      "tokens_seen": 3052470272
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5150056170759108e-05,
+      "loss": 1.3394,
+      "theoretical_loss": 3.344810891267615,
+      "tokens_seen": 3052470272
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5142031776600868e-05,
+      "loss": 1.2352,
+      "theoretical_loss": 3.3447999812141567,
+      "tokens_seen": 3052601344
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5134007382442628e-05,
+      "loss": 1.2042,
+      "theoretical_loss": 3.3447890717603026,
+      "tokens_seen": 3052732416
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5125982988284385e-05,
+      "loss": 1.2329,
+      "theoretical_loss": 3.3447781629059943,
+      "tokens_seen": 3052863488
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5117958594126145e-05,
+      "loss": 1.226,
+      "theoretical_loss": 3.3447672546511726,
+      "tokens_seen": 3052994560
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5109934199967905e-05,
+      "loss": 1.2618,
+      "theoretical_loss": 3.344756346995779,
+      "tokens_seen": 3053125632
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5101909805809661e-05,
+      "loss": 1.3359,
+      "theoretical_loss": 3.3447454399397554,
+      "tokens_seen": 3053256704
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5093885411651421e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.3447345334830425,
+      "tokens_seen": 3053387776
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5085861017493181e-05,
+      "loss": 1.1645,
+      "theoretical_loss": 3.3447236276255823,
+      "tokens_seen": 3053518848
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5077836623334939e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3447127223673156,
+      "tokens_seen": 3053649920
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5069812229176697e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.3447018177081835,
+      "tokens_seen": 3053780992
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5061787835018457e-05,
+      "loss": 1.2319,
+      "theoretical_loss": 3.3446909136481278,
+      "tokens_seen": 3053912064
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5053763440860215e-05,
+      "loss": 1.2716,
+      "theoretical_loss": 3.34468001018709,
+      "tokens_seen": 3054043136
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5045739046701975e-05,
+      "loss": 1.1487,
+      "theoretical_loss": 3.344669107325011,
+      "tokens_seen": 3054174208
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5037714652543734e-05,
+      "loss": 1.2724,
+      "theoretical_loss": 3.344658205061833,
+      "tokens_seen": 3054305280
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5029690258385492e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.3446473033974966,
+      "tokens_seen": 3054436352
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5021665864227252e-05,
+      "loss": 1.1798,
+      "theoretical_loss": 3.3446364023319437,
+      "tokens_seen": 3054567424
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.501364147006901e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.3446255018651154,
+      "tokens_seen": 3054698496
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.5005617075910768e-05,
+      "loss": 1.2013,
+      "theoretical_loss": 3.344614601996953,
+      "tokens_seen": 3054829568
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4997592681752528e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3446037027273987,
+      "tokens_seen": 3054960640
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4989568287594288e-05,
+      "loss": 1.186,
+      "theoretical_loss": 3.3445928040563935,
+      "tokens_seen": 3055091712
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4981543893436044e-05,
+      "loss": 1.2424,
+      "theoretical_loss": 3.3445819059838784,
+      "tokens_seen": 3055222784
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4973519499277804e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3445710085097953,
+      "tokens_seen": 3055353856
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4965495105119564e-05,
+      "loss": 1.206,
+      "theoretical_loss": 3.344560111634086,
+      "tokens_seen": 3055484928
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4957470710961324e-05,
+      "loss": 1.2736,
+      "theoretical_loss": 3.3445492153566914,
+      "tokens_seen": 3055616000
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.494944631680308e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.344538319677553,
+      "tokens_seen": 3055747072
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.494142192264484e-05,
+      "loss": 1.2909,
+      "theoretical_loss": 3.3445274245966123,
+      "tokens_seen": 3055878144
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.49333975284866e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3445165301138116,
+      "tokens_seen": 3056009216
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4925373134328357e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.3445056362290915,
+      "tokens_seen": 3056140288
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4917348740170117e-05,
+      "loss": 1.1877,
+      "theoretical_loss": 3.344494742942394,
+      "tokens_seen": 3056271360
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4909324346011877e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.3444838502536602,
+      "tokens_seen": 3056402432
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4901299951853637e-05,
+      "loss": 1.2508,
+      "theoretical_loss": 3.344472958162832,
+      "tokens_seen": 3056533504
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4893275557695393e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3444620666698506,
+      "tokens_seen": 3056664576
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4885251163537153e-05,
+      "loss": 1.2378,
+      "theoretical_loss": 3.3444511757746582,
+      "tokens_seen": 3056795648
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4877226769378913e-05,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3444402854771957,
+      "tokens_seen": 3056926720
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4869202375220673e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3444293957774045,
+      "tokens_seen": 3057057792
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.486117798106243e-05,
+      "loss": 1.1949,
+      "theoretical_loss": 3.344418506675227,
+      "tokens_seen": 3057188864
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.485315358690419e-05,
+      "loss": 1.2907,
+      "theoretical_loss": 3.344407618170605,
+      "tokens_seen": 3057319936
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.484512919274595e-05,
+      "loss": 1.1713,
+      "theoretical_loss": 3.3443967302634783,
+      "tokens_seen": 3057451008
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4837104798587706e-05,
+      "loss": 1.3125,
+      "theoretical_loss": 3.3443858429537903,
+      "tokens_seen": 3057582080
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4829080404429466e-05,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3443749562414817,
+      "tokens_seen": 3057713152
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4821056010271226e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.3443640701264945,
+      "tokens_seen": 3057844224
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4813031616112986e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.34435318460877,
+      "tokens_seen": 3057975296
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4805007221954742e-05,
+      "loss": 1.2367,
+      "theoretical_loss": 3.3443422996882504,
+      "tokens_seen": 3058106368
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4796982827796502e-05,
+      "loss": 1.1914,
+      "theoretical_loss": 3.344331415364877,
+      "tokens_seen": 3058237440
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4788958433638262e-05,
+      "loss": 1.2912,
+      "theoretical_loss": 3.344320531638591,
+      "tokens_seen": 3058368512
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4780934039480022e-05,
+      "loss": 1.2642,
+      "theoretical_loss": 3.344309648509335,
+      "tokens_seen": 3058499584
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4772909645321778e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.3442987659770496,
+      "tokens_seen": 3058630656
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4764885251163538e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.3442878840416776,
+      "tokens_seen": 3058761728
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4756860857005298e-05,
+      "loss": 1.2435,
+      "theoretical_loss": 3.3442770027031594,
+      "tokens_seen": 3058892800
+    },
+    {
+      "epoch": 0.85,
+      "objective/train/docs_used": 1025165,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2271206378936768,
+      "objective/train/theoretical_loss": 3.344266121961438,
+      "objective/train/tokens_used": 1429482976,
+      "theoretical_loss": 3.344266121961438,
+      "tokens_seen": 3059023872
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4748836462847056e-05,
+      "loss": 1.2215,
+      "theoretical_loss": 3.344266121961438,
+      "tokens_seen": 3059023872
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4740812068688815e-05,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3442552418164544,
+      "tokens_seen": 3059154944
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4732787674530574e-05,
+      "loss": 1.2584,
+      "theoretical_loss": 3.3442443622681504,
+      "tokens_seen": 3059286016
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4724763280372333e-05,
+      "loss": 1.2441,
+      "theoretical_loss": 3.3442334833164677,
+      "tokens_seen": 3059417088
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.471673888621409e-05,
+      "loss": 1.225,
+      "theoretical_loss": 3.344222604961348,
+      "tokens_seen": 3059548160
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.470871449205585e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.344211727202733,
+      "tokens_seen": 3059679232
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4700690097897609e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.3442008500405644,
+      "tokens_seen": 3059810304
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4692665703739369e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.3441899734747844,
+      "tokens_seen": 3059941376
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4684641309581127e-05,
+      "loss": 1.3246,
+      "theoretical_loss": 3.344179097505334,
+      "tokens_seen": 3060072448
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4676616915422885e-05,
+      "loss": 1.2119,
+      "theoretical_loss": 3.344168222132156,
+      "tokens_seen": 3060203520
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4668592521264645e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.3441573473551913,
+      "tokens_seen": 3060334592
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4660568127106405e-05,
+      "loss": 1.2794,
+      "theoretical_loss": 3.3441464731743817,
+      "tokens_seen": 3060465664
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.4652543732948162e-05,
+      "loss": 1.2309,
+      "theoretical_loss": 3.3441355995896695,
+      "tokens_seen": 3060596736
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4644519338789922e-05,
+      "loss": 1.2519,
+      "theoretical_loss": 3.344124726600996,
+      "tokens_seen": 3060727808
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4636494944631681e-05,
+      "loss": 1.2894,
+      "theoretical_loss": 3.3441138542083033,
+      "tokens_seen": 3060858880
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4628470550473438e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.3441029824115334,
+      "tokens_seen": 3060989952
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4620446156315198e-05,
+      "loss": 1.2717,
+      "theoretical_loss": 3.3440921112106277,
+      "tokens_seen": 3061121024
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4612421762156958e-05,
+      "loss": 1.2927,
+      "theoretical_loss": 3.344081240605528,
+      "tokens_seen": 3061252096
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4604397367998718e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3440703705961767,
+      "tokens_seen": 3061383168
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4596372973840474e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.344059501182515,
+      "tokens_seen": 3061514240
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4588348579682234e-05,
+      "loss": 1.2244,
+      "theoretical_loss": 3.344048632364485,
+      "tokens_seen": 3061645312
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4580324185523994e-05,
+      "loss": 1.1953,
+      "theoretical_loss": 3.344037764142029,
+      "tokens_seen": 3061776384
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4572299791365754e-05,
+      "loss": 1.263,
+      "theoretical_loss": 3.3440268965150883,
+      "tokens_seen": 3061907456
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.456427539720751e-05,
+      "loss": 1.2376,
+      "theoretical_loss": 3.3440160294836048,
+      "tokens_seen": 3062038528
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.455625100304927e-05,
+      "loss": 1.3006,
+      "theoretical_loss": 3.3440051630475205,
+      "tokens_seen": 3062169600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.454822660889103e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.3439942972067773,
+      "tokens_seen": 3062300672
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4540202214732787e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.343983431961317,
+      "tokens_seen": 3062431744
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4532177820574547e-05,
+      "loss": 1.1713,
+      "theoretical_loss": 3.343972567311082,
+      "tokens_seen": 3062562816
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4524153426416307e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.343961703256014,
+      "tokens_seen": 3062693888
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4516129032258066e-05,
+      "loss": 1.2769,
+      "theoretical_loss": 3.3439508397960545,
+      "tokens_seen": 3062824960
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4508104638099823e-05,
+      "loss": 1.2995,
+      "theoretical_loss": 3.343939976931146,
+      "tokens_seen": 3062956032
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4500080243941583e-05,
+      "loss": 1.2125,
+      "theoretical_loss": 3.34392911466123,
+      "tokens_seen": 3063087104
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4492055849783343e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3439182529862483,
+      "tokens_seen": 3063218176
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4484031455625103e-05,
+      "loss": 1.2368,
+      "theoretical_loss": 3.3439073919061437,
+      "tokens_seen": 3063349248
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.447600706146686e-05,
+      "loss": 1.283,
+      "theoretical_loss": 3.3438965314208575,
+      "tokens_seen": 3063480320
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4467982667308619e-05,
+      "loss": 1.1422,
+      "theoretical_loss": 3.3438856715303316,
+      "tokens_seen": 3063611392
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4459958273150379e-05,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3438748122345086,
+      "tokens_seen": 3063742464
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4451933878992136e-05,
+      "loss": 1.299,
+      "theoretical_loss": 3.3438639535333294,
+      "tokens_seen": 3063873536
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4443909484833895e-05,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3438530954267374,
+      "tokens_seen": 3064004608
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4435885090675655e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3438422379146737,
+      "tokens_seen": 3064135680
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4427860696517415e-05,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3438313809970808,
+      "tokens_seen": 3064266752
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4419836302359172e-05,
+      "loss": 1.3513,
+      "theoretical_loss": 3.3438205246739,
+      "tokens_seen": 3064397824
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4411811908200932e-05,
+      "loss": 1.2115,
+      "theoretical_loss": 3.343809668945074,
+      "tokens_seen": 3064528896
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4403787514042692e-05,
+      "loss": 1.2152,
+      "theoretical_loss": 3.343798813810545,
+      "tokens_seen": 3064659968
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.439576311988445e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.343787959270254,
+      "tokens_seen": 3064791040
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4387738725726208e-05,
+      "loss": 1.3017,
+      "theoretical_loss": 3.343777105324144,
+      "tokens_seen": 3064922112
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4379714331567968e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.343766251972157,
+      "tokens_seen": 3065053184
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4371689937409726e-05,
+      "loss": 1.1738,
+      "theoretical_loss": 3.343755399214235,
+      "tokens_seen": 3065184256
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4363665543251484e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3437445470503193,
+      "tokens_seen": 3065315328
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4355641149093244e-05,
+      "loss": 1.2552,
+      "theoretical_loss": 3.343733695480353,
+      "tokens_seen": 3065446400
+    },
+    {
+      "epoch": 0.86,
+      "objective/train/docs_used": 1027393,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1943154335021973,
+      "objective/train/theoretical_loss": 3.343722844504278,
+      "objective/train/tokens_used": 1436036576,
+      "theoretical_loss": 3.343722844504278,
+      "tokens_seen": 3065577472
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4347616754935002e-05,
+      "loss": 1.1864,
+      "theoretical_loss": 3.343722844504278,
+      "tokens_seen": 3065577472
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4339592360776762e-05,
+      "loss": 1.23,
+      "theoretical_loss": 3.3437119941220366,
+      "tokens_seen": 3065708544
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.433156796661852e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3437011443335702,
+      "tokens_seen": 3065839616
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4323543572460279e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3436902951388214,
+      "tokens_seen": 3065970688
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4315519178302039e-05,
+      "loss": 1.2952,
+      "theoretical_loss": 3.343679446537732,
+      "tokens_seen": 3066101760
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4307494784143799e-05,
+      "loss": 1.3353,
+      "theoretical_loss": 3.3436685985302446,
+      "tokens_seen": 3066232832
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4299470389985555e-05,
+      "loss": 1.1932,
+      "theoretical_loss": 3.343657751116301,
+      "tokens_seen": 3066363904
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4291445995827315e-05,
+      "loss": 1.3091,
+      "theoretical_loss": 3.3436469042958437,
+      "tokens_seen": 3066494976
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4283421601669075e-05,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3436360580688143,
+      "tokens_seen": 3066626048
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4275397207510835e-05,
+      "loss": 1.2673,
+      "theoretical_loss": 3.3436252124351555,
+      "tokens_seen": 3066757120
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4267372813352591e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3436143673948093,
+      "tokens_seen": 3066888192
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4259348419194351e-05,
+      "loss": 1.1956,
+      "theoretical_loss": 3.343603522947718,
+      "tokens_seen": 3067019264
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4251324025036111e-05,
+      "loss": 1.2105,
+      "theoretical_loss": 3.343592679093823,
+      "tokens_seen": 3067150336
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4243299630877868e-05,
+      "loss": 1.2513,
+      "theoretical_loss": 3.343581835833068,
+      "tokens_seen": 3067281408
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4235275236719628e-05,
+      "loss": 1.2122,
+      "theoretical_loss": 3.343570993165394,
+      "tokens_seen": 3067412480
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4227250842561388e-05,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3435601510907436,
+      "tokens_seen": 3067543552
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4219226448403147e-05,
+      "loss": 1.2317,
+      "theoretical_loss": 3.3435493096090596,
+      "tokens_seen": 3067674624
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4211202054244904e-05,
+      "loss": 1.1778,
+      "theoretical_loss": 3.3435384687202827,
+      "tokens_seen": 3067805696
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4203177660086664e-05,
+      "loss": 1.2504,
+      "theoretical_loss": 3.343527628424357,
+      "tokens_seen": 3067936768
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4195153265928424e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.3435167887212236,
+      "tokens_seen": 3068067840
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4187128871770184e-05,
+      "loss": 1.2423,
+      "theoretical_loss": 3.343505949610825,
+      "tokens_seen": 3068198912
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.417910447761194e-05,
+      "loss": 1.2492,
+      "theoretical_loss": 3.343495111093103,
+      "tokens_seen": 3068329984
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.41710800834537e-05,
+      "loss": 1.2043,
+      "theoretical_loss": 3.3434842731680012,
+      "tokens_seen": 3068461056
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.416305568929546e-05,
+      "loss": 1.306,
+      "theoretical_loss": 3.3434734358354605,
+      "tokens_seen": 3068592128
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4155031295137216e-05,
+      "loss": 1.2464,
+      "theoretical_loss": 3.343462599095424,
+      "tokens_seen": 3068723200
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4147006900978976e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.3434517629478337,
+      "tokens_seen": 3068854272
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4138982506820736e-05,
+      "loss": 1.2802,
+      "theoretical_loss": 3.343440927392632,
+      "tokens_seen": 3068985344
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4130958112662496e-05,
+      "loss": 1.2631,
+      "theoretical_loss": 3.3434300924297613,
+      "tokens_seen": 3069116416
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4122933718504253e-05,
+      "loss": 1.2431,
+      "theoretical_loss": 3.3434192580591637,
+      "tokens_seen": 3069247488
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4114909324346013e-05,
+      "loss": 1.1843,
+      "theoretical_loss": 3.3434084242807818,
+      "tokens_seen": 3069378560
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4106884930187773e-05,
+      "loss": 1.185,
+      "theoretical_loss": 3.3433975910945577,
+      "tokens_seen": 3069509632
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4098860536029532e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.3433867585004338,
+      "tokens_seen": 3069640704
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4090836141871289e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3433759264983522,
+      "tokens_seen": 3069771776
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4082811747713049e-05,
+      "loss": 1.2076,
+      "theoretical_loss": 3.343365095088256,
+      "tokens_seen": 3069902848
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4074787353554809e-05,
+      "loss": 1.1728,
+      "theoretical_loss": 3.343354264270087,
+      "tokens_seen": 3070033920
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4066762959396565e-05,
+      "loss": 1.2732,
+      "theoretical_loss": 3.3433434340437875,
+      "tokens_seen": 3070164992
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4058738565238325e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.3433326044093,
+      "tokens_seen": 3070296064
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4050714171080085e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.343321775366568,
+      "tokens_seen": 3070427136
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4042689776921843e-05,
+      "loss": 1.2941,
+      "theoretical_loss": 3.3433109469155315,
+      "tokens_seen": 3070558208
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4034665382763602e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3433001190561353,
+      "tokens_seen": 3070689280
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4026640988605361e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.3432892917883206,
+      "tokens_seen": 3070820352
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.401861659444712e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.34327846511203,
+      "tokens_seen": 3070951424
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.401059220028888e-05,
+      "loss": 1.2687,
+      "theoretical_loss": 3.3432676390272063,
+      "tokens_seen": 3071082496
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.4002567806130638e-05,
+      "loss": 1.2686,
+      "theoretical_loss": 3.3432568135337912,
+      "tokens_seen": 3071213568
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3994543411972396e-05,
+      "loss": 1.2572,
+      "theoretical_loss": 3.343245988631728,
+      "tokens_seen": 3071344640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3986519017814156e-05,
+      "loss": 1.2976,
+      "theoretical_loss": 3.3432351643209586,
+      "tokens_seen": 3071475712
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3978494623655914e-05,
+      "loss": 1.2897,
+      "theoretical_loss": 3.343224340601426,
+      "tokens_seen": 3071606784
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3970470229497672e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.343213517473072,
+      "tokens_seen": 3071737856
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3962445835339432e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.3432026949358393,
+      "tokens_seen": 3071868928
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3954421441181192e-05,
+      "loss": 1.2932,
+      "theoretical_loss": 3.343191872989671,
+      "tokens_seen": 3072000000
+    },
+    {
+      "epoch": 0.86,
+      "objective/train/docs_used": 1029784,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1841344833374023,
+      "objective/train/theoretical_loss": 3.3431810516345086,
+      "objective/train/tokens_used": 1442590176,
+      "theoretical_loss": 3.3431810516345086,
+      "tokens_seen": 3072131072
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3946397047022949e-05,
+      "loss": 1.2356,
+      "theoretical_loss": 3.3431810516345086,
+      "tokens_seen": 3072131072
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3938372652864709e-05,
+      "loss": 1.2976,
+      "theoretical_loss": 3.343170230870295,
+      "tokens_seen": 3072262144
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3930348258706468e-05,
+      "loss": 1.2654,
+      "theoretical_loss": 3.3431594106969733,
+      "tokens_seen": 3072393216
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3922323864548228e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.3431485911144856,
+      "tokens_seen": 3072524288
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3914299470389985e-05,
+      "loss": 1.3094,
+      "theoretical_loss": 3.343137772122774,
+      "tokens_seen": 3072655360
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3906275076231745e-05,
+      "loss": 1.281,
+      "theoretical_loss": 3.343126953721782,
+      "tokens_seen": 3072786432
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3898250682073505e-05,
+      "loss": 1.2796,
+      "theoretical_loss": 3.3431161359114507,
+      "tokens_seen": 3072917504
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3890226287915261e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.3431053186917246,
+      "tokens_seen": 3073048576
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3882201893757021e-05,
+      "loss": 1.2307,
+      "theoretical_loss": 3.3430945020625447,
+      "tokens_seen": 3073179648
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3874177499598781e-05,
+      "loss": 1.1922,
+      "theoretical_loss": 3.343083686023854,
+      "tokens_seen": 3073310720
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3866153105440541e-05,
+      "loss": 1.2447,
+      "theoretical_loss": 3.3430728705755954,
+      "tokens_seen": 3073441792
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3858128711282297e-05,
+      "loss": 1.2774,
+      "theoretical_loss": 3.343062055717711,
+      "tokens_seen": 3073572864
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3850104317124057e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.3430512414501443,
+      "tokens_seen": 3073703936
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3842079922965817e-05,
+      "loss": 1.3133,
+      "theoretical_loss": 3.343040427772837,
+      "tokens_seen": 3073835008
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3834055528807577e-05,
+      "loss": 1.2924,
+      "theoretical_loss": 3.3430296146857317,
+      "tokens_seen": 3073966080
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3826031134649334e-05,
+      "loss": 1.177,
+      "theoretical_loss": 3.3430188021887712,
+      "tokens_seen": 3074097152
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3818006740491094e-05,
+      "loss": 1.2287,
+      "theoretical_loss": 3.3430079902818988,
+      "tokens_seen": 3074228224
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3809982346332853e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.3429971789650565,
+      "tokens_seen": 3074359296
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3801957952174613e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.342986368238187,
+      "tokens_seen": 3074490368
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.379393355801637e-05,
+      "loss": 1.2949,
+      "theoretical_loss": 3.342975558101233,
+      "tokens_seen": 3074621440
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.378590916385813e-05,
+      "loss": 1.2625,
+      "theoretical_loss": 3.3429647485541367,
+      "tokens_seen": 3074752512
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.377788476969989e-05,
+      "loss": 1.2742,
+      "theoretical_loss": 3.3429539395968417,
+      "tokens_seen": 3074883584
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3769860375541646e-05,
+      "loss": 1.2437,
+      "theoretical_loss": 3.34294313122929,
+      "tokens_seen": 3075014656
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3761835981383406e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.3429323234514245,
+      "tokens_seen": 3075145728
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3753811587225166e-05,
+      "loss": 1.2874,
+      "theoretical_loss": 3.3429215162631882,
+      "tokens_seen": 3075276800
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3745787193066926e-05,
+      "loss": 1.3158,
+      "theoretical_loss": 3.3429107096645234,
+      "tokens_seen": 3075407872
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3737762798908682e-05,
+      "loss": 1.1833,
+      "theoretical_loss": 3.342899903655373,
+      "tokens_seen": 3075538944
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3729738404750442e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.342889098235679,
+      "tokens_seen": 3075670016
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3721714010592202e-05,
+      "loss": 1.2277,
+      "theoretical_loss": 3.342878293405385,
+      "tokens_seen": 3075801088
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.371368961643396e-05,
+      "loss": 1.2935,
+      "theoretical_loss": 3.342867489164434,
+      "tokens_seen": 3075932160
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3705665222275719e-05,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3428566855127677,
+      "tokens_seen": 3076063232
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3697640828117479e-05,
+      "loss": 1.32,
+      "theoretical_loss": 3.3428458824503298,
+      "tokens_seen": 3076194304
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3689616433959237e-05,
+      "loss": 1.2156,
+      "theoretical_loss": 3.3428350799770623,
+      "tokens_seen": 3076325376
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3681592039800995e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.3428242780929085,
+      "tokens_seen": 3076456448
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3673567645642755e-05,
+      "loss": 1.2333,
+      "theoretical_loss": 3.342813476797811,
+      "tokens_seen": 3076587520
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3665543251484513e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3428026760917127,
+      "tokens_seen": 3076718592
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3657518857326273e-05,
+      "loss": 1.207,
+      "theoretical_loss": 3.342791875974556,
+      "tokens_seen": 3076849664
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.3649494463168031e-05,
+      "loss": 1.1856,
+      "theoretical_loss": 3.342781076446284,
+      "tokens_seen": 3076980736
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.364147006900979e-05,
+      "loss": 1.2936,
+      "theoretical_loss": 3.3427702775068395,
+      "tokens_seen": 3077111808
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.363344567485155e-05,
+      "loss": 1.2612,
+      "theoretical_loss": 3.3427594791561654,
+      "tokens_seen": 3077242880
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.362542128069331e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3427486813942044,
+      "tokens_seen": 3077373952
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3617396886535066e-05,
+      "loss": 1.2929,
+      "theoretical_loss": 3.3427378842208992,
+      "tokens_seen": 3077505024
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3609372492376826e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.342727087636193,
+      "tokens_seen": 3077636096
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3601348098218586e-05,
+      "loss": 1.2112,
+      "theoretical_loss": 3.3427162916400284,
+      "tokens_seen": 3077767168
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3593323704060342e-05,
+      "loss": 1.2988,
+      "theoretical_loss": 3.342705496232348,
+      "tokens_seen": 3077898240
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3585299309902102e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.3426947014130954,
+      "tokens_seen": 3078029312
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3577274915743862e-05,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3426839071822125,
+      "tokens_seen": 3078160384
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3569250521585622e-05,
+      "loss": 1.2312,
+      "theoretical_loss": 3.342673113539643,
+      "tokens_seen": 3078291456
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3561226127427378e-05,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3426623204853296,
+      "tokens_seen": 3078422528
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3553201733269138e-05,
+      "loss": 1.2432,
+      "theoretical_loss": 3.342651528019215,
+      "tokens_seen": 3078553600
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1031977,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.297423005104065,
+      "objective/train/theoretical_loss": 3.342640736141242,
+      "objective/train/tokens_used": 1449143776,
+      "theoretical_loss": 3.342640736141242,
+      "tokens_seen": 3078684672
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3545177339110898e-05,
+      "loss": 1.268,
+      "theoretical_loss": 3.342640736141242,
+      "tokens_seen": 3078684672
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3537152944952658e-05,
+      "loss": 1.228,
+      "theoretical_loss": 3.3426299448513537,
+      "tokens_seen": 3078815744
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3529128550794415e-05,
+      "loss": 1.2264,
+      "theoretical_loss": 3.342619154149493,
+      "tokens_seen": 3078946816
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3521104156636175e-05,
+      "loss": 1.2977,
+      "theoretical_loss": 3.3426083640356032,
+      "tokens_seen": 3079077888
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3513079762477934e-05,
+      "loss": 1.2236,
+      "theoretical_loss": 3.3425975745096266,
+      "tokens_seen": 3079208960
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3505055368319691e-05,
+      "loss": 1.2324,
+      "theoretical_loss": 3.3425867855715063,
+      "tokens_seen": 3079340032
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3497030974161451e-05,
+      "loss": 1.2966,
+      "theoretical_loss": 3.3425759972211857,
+      "tokens_seen": 3079471104
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.348900658000321e-05,
+      "loss": 1.1849,
+      "theoretical_loss": 3.3425652094586074,
+      "tokens_seen": 3079602176
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.348098218584497e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.3425544222837145,
+      "tokens_seen": 3079733248
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3472957791686727e-05,
+      "loss": 1.1901,
+      "theoretical_loss": 3.3425436356964493,
+      "tokens_seen": 3079864320
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3464933397528487e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.342532849696756,
+      "tokens_seen": 3079995392
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3456909003370247e-05,
+      "loss": 1.2906,
+      "theoretical_loss": 3.3425220642845765,
+      "tokens_seen": 3080126464
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3448884609212007e-05,
+      "loss": 1.241,
+      "theoretical_loss": 3.3425112794598544,
+      "tokens_seen": 3080257536
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3440860215053763e-05,
+      "loss": 1.2515,
+      "theoretical_loss": 3.3425004952225326,
+      "tokens_seen": 3080388608
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3432835820895523e-05,
+      "loss": 1.2243,
+      "theoretical_loss": 3.3424897115725543,
+      "tokens_seen": 3080519680
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3424811426737283e-05,
+      "loss": 1.3101,
+      "theoretical_loss": 3.3424789285098617,
+      "tokens_seen": 3080650752
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.341678703257904e-05,
+      "loss": 1.1837,
+      "theoretical_loss": 3.342468146034399,
+      "tokens_seen": 3080781824
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.34087626384208e-05,
+      "loss": 1.2721,
+      "theoretical_loss": 3.3424573641461084,
+      "tokens_seen": 3080912896
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.340073824426256e-05,
+      "loss": 1.2636,
+      "theoretical_loss": 3.342446582844933,
+      "tokens_seen": 3081043968
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.339271385010432e-05,
+      "loss": 1.2665,
+      "theoretical_loss": 3.3424358021308165,
+      "tokens_seen": 3081175040
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3384689455946076e-05,
+      "loss": 1.2715,
+      "theoretical_loss": 3.342425022003701,
+      "tokens_seen": 3081306112
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3376665061787836e-05,
+      "loss": 1.2279,
+      "theoretical_loss": 3.342414242463531,
+      "tokens_seen": 3081437184
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3368640667629596e-05,
+      "loss": 1.2001,
+      "theoretical_loss": 3.342403463510248,
+      "tokens_seen": 3081568256
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3360616273471354e-05,
+      "loss": 1.2532,
+      "theoretical_loss": 3.3423926851437957,
+      "tokens_seen": 3081699328
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3352591879313112e-05,
+      "loss": 1.2297,
+      "theoretical_loss": 3.342381907364117,
+      "tokens_seen": 3081830400
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3344567485154872e-05,
+      "loss": 1.3267,
+      "theoretical_loss": 3.342371130171156,
+      "tokens_seen": 3081961472
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.333654309099663e-05,
+      "loss": 1.2645,
+      "theoretical_loss": 3.342360353564855,
+      "tokens_seen": 3082092544
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.332851869683839e-05,
+      "loss": 1.2451,
+      "theoretical_loss": 3.3423495775451566,
+      "tokens_seen": 3082223616
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3320494302680148e-05,
+      "loss": 1.2606,
+      "theoretical_loss": 3.342338802112005,
+      "tokens_seen": 3082354688
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3312469908521907e-05,
+      "loss": 1.196,
+      "theoretical_loss": 3.3423280272653426,
+      "tokens_seen": 3082485760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3304445514363667e-05,
+      "loss": 1.257,
+      "theoretical_loss": 3.3423172530051133,
+      "tokens_seen": 3082616832
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3296421120205425e-05,
+      "loss": 1.2853,
+      "theoretical_loss": 3.342306479331259,
+      "tokens_seen": 3082747904
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3288396726047183e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.342295706243724,
+      "tokens_seen": 3082878976
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3280372331888943e-05,
+      "loss": 1.2041,
+      "theoretical_loss": 3.3422849337424507,
+      "tokens_seen": 3083010048
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3272347937730703e-05,
+      "loss": 1.2571,
+      "theoretical_loss": 3.3422741618273832,
+      "tokens_seen": 3083141120
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.326432354357246e-05,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3422633904984638,
+      "tokens_seen": 3083272192
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.325629914941422e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.342252619755636,
+      "tokens_seen": 3083403264
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3248274755255979e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.3422418495988433,
+      "tokens_seen": 3083534336
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3240250361097739e-05,
+      "loss": 1.2304,
+      "theoretical_loss": 3.342231080028028,
+      "tokens_seen": 3083665408
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3232225966939496e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3422203110431346,
+      "tokens_seen": 3083796480
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3224201572781255e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.342209542644105,
+      "tokens_seen": 3083927552
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3216177178623015e-05,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3421987748308837,
+      "tokens_seen": 3084058624
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3208152784464772e-05,
+      "loss": 1.2422,
+      "theoretical_loss": 3.3421880076034127,
+      "tokens_seen": 3084189696
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3200128390306532e-05,
+      "loss": 1.1477,
+      "theoretical_loss": 3.3421772409616364,
+      "tokens_seen": 3084320768
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3192103996148292e-05,
+      "loss": 1.2793,
+      "theoretical_loss": 3.3421664749054973,
+      "tokens_seen": 3084451840
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3184079601990052e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.342155709434939,
+      "tokens_seen": 3084582912
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3176055207831808e-05,
+      "loss": 1.2414,
+      "theoretical_loss": 3.3421449445499043,
+      "tokens_seen": 3084713984
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3168030813673568e-05,
+      "loss": 1.1648,
+      "theoretical_loss": 3.342134180250337,
+      "tokens_seen": 3084845056
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3160006419515328e-05,
+      "loss": 1.1925,
+      "theoretical_loss": 3.34212341653618,
+      "tokens_seen": 3084976128
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3151982025357088e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.3421126534073773,
+      "tokens_seen": 3085107200
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1034060,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2990813255310059,
+      "objective/train/theoretical_loss": 3.342101890863871,
+      "objective/train/tokens_used": 1455697376,
+      "theoretical_loss": 3.342101890863871,
+      "tokens_seen": 3085238272
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3143957631198844e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.342101890863871,
+      "tokens_seen": 3085238272
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3135933237040604e-05,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3420911289056057,
+      "tokens_seen": 3085369344
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3127908842882364e-05,
+      "loss": 1.331,
+      "theoretical_loss": 3.3420803675325237,
+      "tokens_seen": 3085500416
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.311988444872412e-05,
+      "loss": 1.2374,
+      "theoretical_loss": 3.342069606744569,
+      "tokens_seen": 3085631488
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.311186005456588e-05,
+      "loss": 1.1784,
+      "theoretical_loss": 3.3420588465416845,
+      "tokens_seen": 3085762560
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.310383566040764e-05,
+      "loss": 1.2132,
+      "theoretical_loss": 3.342048086923814,
+      "tokens_seen": 3085893632
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.30958112662494e-05,
+      "loss": 1.247,
+      "theoretical_loss": 3.3420373278909006,
+      "tokens_seen": 3086024704
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3087786872091157e-05,
+      "loss": 1.3214,
+      "theoretical_loss": 3.3420265694428872,
+      "tokens_seen": 3086155776
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3079762477932917e-05,
+      "loss": 1.2786,
+      "theoretical_loss": 3.3420158115797176,
+      "tokens_seen": 3086286848
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3071738083774677e-05,
+      "loss": 1.1911,
+      "theoretical_loss": 3.3420050543013353,
+      "tokens_seen": 3086417920
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3063713689616437e-05,
+      "loss": 1.2444,
+      "theoretical_loss": 3.3419942976076835,
+      "tokens_seen": 3086548992
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3055689295458193e-05,
+      "loss": 1.2295,
+      "theoretical_loss": 3.341983541498706,
+      "tokens_seen": 3086680064
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3047664901299953e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.3419727859743453,
+      "tokens_seen": 3086811136
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3039640507141713e-05,
+      "loss": 1.2068,
+      "theoretical_loss": 3.3419620310345453,
+      "tokens_seen": 3086942208
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.303161611298347e-05,
+      "loss": 1.2078,
+      "theoretical_loss": 3.34195127667925,
+      "tokens_seen": 3087073280
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.302359171882523e-05,
+      "loss": 1.2117,
+      "theoretical_loss": 3.3419405229084016,
+      "tokens_seen": 3087204352
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3015567324666988e-05,
+      "loss": 1.2121,
+      "theoretical_loss": 3.3419297697219443,
+      "tokens_seen": 3087335424
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.3007542930508748e-05,
+      "loss": 1.2574,
+      "theoretical_loss": 3.341919017119822,
+      "tokens_seen": 3087466496
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2999518536350506e-05,
+      "loss": 1.2388,
+      "theoretical_loss": 3.341908265101977,
+      "tokens_seen": 3087597568
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2991494142192264e-05,
+      "loss": 1.2458,
+      "theoretical_loss": 3.341897513668353,
+      "tokens_seen": 3087728640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2983469748034024e-05,
+      "loss": 1.2694,
+      "theoretical_loss": 3.341886762818894,
+      "tokens_seen": 3087859712
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2975445353875784e-05,
+      "loss": 1.215,
+      "theoretical_loss": 3.3418760125535436,
+      "tokens_seen": 3087990784
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.296742095971754e-05,
+      "loss": 1.2828,
+      "theoretical_loss": 3.3418652628722443,
+      "tokens_seen": 3088121856
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.29593965655593e-05,
+      "loss": 1.1879,
+      "theoretical_loss": 3.3418545137749405,
+      "tokens_seen": 3088252928
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.295137217140106e-05,
+      "loss": 1.2994,
+      "theoretical_loss": 3.3418437652615753,
+      "tokens_seen": 3088384000
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2943347777242817e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3418330173320925,
+      "tokens_seen": 3088515072
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2935323383084577e-05,
+      "loss": 1.2854,
+      "theoretical_loss": 3.341822269986435,
+      "tokens_seen": 3088646144
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2927298988926336e-05,
+      "loss": 1.2083,
+      "theoretical_loss": 3.341811523224547,
+      "tokens_seen": 3088777216
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2919274594768096e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.341800777046372,
+      "tokens_seen": 3088908288
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2911250200609853e-05,
+      "loss": 1.1908,
+      "theoretical_loss": 3.3417900314518523,
+      "tokens_seen": 3089039360
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2903225806451613e-05,
+      "loss": 1.2385,
+      "theoretical_loss": 3.341779286440933,
+      "tokens_seen": 3089170432
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2895201412293373e-05,
+      "loss": 1.2707,
+      "theoretical_loss": 3.3417685420135568,
+      "tokens_seen": 3089301504
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2887177018135133e-05,
+      "loss": 1.1989,
+      "theoretical_loss": 3.3417577981696676,
+      "tokens_seen": 3089432576
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2879152623976889e-05,
+      "loss": 1.1976,
+      "theoretical_loss": 3.3417470549092085,
+      "tokens_seen": 3089563648
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2871128229818649e-05,
+      "loss": 1.1426,
+      "theoretical_loss": 3.3417363122321238,
+      "tokens_seen": 3089694720
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2863103835660409e-05,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3417255701383564,
+      "tokens_seen": 3089825792
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2855079441502169e-05,
+      "loss": 1.1888,
+      "theoretical_loss": 3.3417148286278504,
+      "tokens_seen": 3089956864
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2847055047343925e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.341704087700549,
+      "tokens_seen": 3090087936
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2839030653185685e-05,
+      "loss": 1.1918,
+      "theoretical_loss": 3.3416933473563963,
+      "tokens_seen": 3090219008
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2831006259027445e-05,
+      "loss": 1.2189,
+      "theoretical_loss": 3.341682607595335,
+      "tokens_seen": 3090350080
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2822981864869202e-05,
+      "loss": 1.1844,
+      "theoretical_loss": 3.3416718684173095,
+      "tokens_seen": 3090481152
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2814957470710962e-05,
+      "loss": 1.2242,
+      "theoretical_loss": 3.341661129822263,
+      "tokens_seen": 3090612224
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2806933076552721e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.3416503918101395,
+      "tokens_seen": 3090743296
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2798908682394481e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3416396543808826,
+      "tokens_seen": 3090874368
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2790884288236238e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.3416289175344356,
+      "tokens_seen": 3091005440
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2782859894077998e-05,
+      "loss": 1.2582,
+      "theoretical_loss": 3.3416181812707424,
+      "tokens_seen": 3091136512
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2774835499919758e-05,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3416074455897466,
+      "tokens_seen": 3091267584
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2766811105761518e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.341596710491392,
+      "tokens_seen": 3091398656
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2758786711603274e-05,
+      "loss": 1.2062,
+      "theoretical_loss": 3.341585975975622,
+      "tokens_seen": 3091529728
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2750762317445034e-05,
+      "loss": 1.2326,
+      "theoretical_loss": 3.3415752420423805,
+      "tokens_seen": 3091660800
+    },
+    {
+      "epoch": 0.87,
+      "objective/train/docs_used": 1036380,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1785664558410645,
+      "objective/train/theoretical_loss": 3.341564508691611,
+      "objective/train/tokens_used": 1462250976,
+      "theoretical_loss": 3.341564508691611,
+      "tokens_seen": 3091791872
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2742737923286794e-05,
+      "loss": 1.175,
+      "theoretical_loss": 3.341564508691611,
+      "tokens_seen": 3091791872
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.273471352912855e-05,
+      "loss": 1.2303,
+      "theoretical_loss": 3.3415537759232574,
+      "tokens_seen": 3091922944
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.272668913497031e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.3415430437372637,
+      "tokens_seen": 3092054016
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.271866474081207e-05,
+      "loss": 1.2191,
+      "theoretical_loss": 3.341532312133573,
+      "tokens_seen": 3092185088
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2710640346653828e-05,
+      "loss": 1.1752,
+      "theoretical_loss": 3.3415215811121293,
+      "tokens_seen": 3092316160
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2702615952495587e-05,
+      "loss": 1.3335,
+      "theoretical_loss": 3.3415108506728766,
+      "tokens_seen": 3092447232
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2694591558337347e-05,
+      "loss": 1.3065,
+      "theoretical_loss": 3.341500120815758,
+      "tokens_seen": 3092578304
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2686567164179105e-05,
+      "loss": 1.277,
+      "theoretical_loss": 3.341489391540718,
+      "tokens_seen": 3092709376
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2678542770020865e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.3414786628476993,
+      "tokens_seen": 3092840448
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2670518375862623e-05,
+      "loss": 1.2305,
+      "theoretical_loss": 3.341467934736647,
+      "tokens_seen": 3092971520
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2662493981704381e-05,
+      "loss": 1.234,
+      "theoretical_loss": 3.3414572072075037,
+      "tokens_seen": 3093102592
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2654469587546141e-05,
+      "loss": 1.2541,
+      "theoretical_loss": 3.341446480260214,
+      "tokens_seen": 3093233664
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.26464451933879e-05,
+      "loss": 1.2176,
+      "theoretical_loss": 3.3414357538947215,
+      "tokens_seen": 3093364736
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2638420799229657e-05,
+      "loss": 1.2035,
+      "theoretical_loss": 3.3414250281109696,
+      "tokens_seen": 3093495808
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.2630396405071417e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.3414143029089027,
+      "tokens_seen": 3093626880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2622372010913177e-05,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3414035782884643,
+      "tokens_seen": 3093757952
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2614347616754934e-05,
+      "loss": 1.1524,
+      "theoretical_loss": 3.341392854249598,
+      "tokens_seen": 3093889024
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2606323222596694e-05,
+      "loss": 1.2235,
+      "theoretical_loss": 3.3413821307922476,
+      "tokens_seen": 3094020096
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2598298828438454e-05,
+      "loss": 1.1958,
+      "theoretical_loss": 3.3413714079163577,
+      "tokens_seen": 3094151168
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2590274434280214e-05,
+      "loss": 1.2454,
+      "theoretical_loss": 3.341360685621871,
+      "tokens_seen": 3094282240
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.258225004012197e-05,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3413499639087325,
+      "tokens_seen": 3094413312
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.257422564596373e-05,
+      "loss": 1.2328,
+      "theoretical_loss": 3.3413392427768853,
+      "tokens_seen": 3094544384
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.256620125180549e-05,
+      "loss": 1.2408,
+      "theoretical_loss": 3.3413285222262736,
+      "tokens_seen": 3094675456
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2558176857647246e-05,
+      "loss": 1.2709,
+      "theoretical_loss": 3.341317802256841,
+      "tokens_seen": 3094806528
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2550152463489006e-05,
+      "loss": 1.295,
+      "theoretical_loss": 3.3413070828685316,
+      "tokens_seen": 3094937600
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2542128069330766e-05,
+      "loss": 1.2372,
+      "theoretical_loss": 3.3412963640612894,
+      "tokens_seen": 3095068672
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2534103675172526e-05,
+      "loss": 1.1897,
+      "theoretical_loss": 3.341285645835058,
+      "tokens_seen": 3095199744
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2526079281014283e-05,
+      "loss": 1.1491,
+      "theoretical_loss": 3.3412749281897813,
+      "tokens_seen": 3095330816
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2518054886856043e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.341264211125403,
+      "tokens_seen": 3095461888
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2510030492697802e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.341253494641868,
+      "tokens_seen": 3095592960
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2502006098539562e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3412427787391197,
+      "tokens_seen": 3095724032
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.249398170438132e-05,
+      "loss": 1.2669,
+      "theoretical_loss": 3.341232063417101,
+      "tokens_seen": 3095855104
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2485957310223079e-05,
+      "loss": 1.217,
+      "theoretical_loss": 3.3412213486757576,
+      "tokens_seen": 3095986176
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2477932916064839e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.3412106345150323,
+      "tokens_seen": 3096117248
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2469908521906597e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3411999209348693,
+      "tokens_seen": 3096248320
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2461884127748355e-05,
+      "loss": 1.1883,
+      "theoretical_loss": 3.341189207935213,
+      "tokens_seen": 3096379392
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2453859733590115e-05,
+      "loss": 1.1829,
+      "theoretical_loss": 3.3411784955160067,
+      "tokens_seen": 3096510464
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2445835339431873e-05,
+      "loss": 1.2918,
+      "theoretical_loss": 3.3411677836771947,
+      "tokens_seen": 3096641536
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2437810945273633e-05,
+      "loss": 1.2899,
+      "theoretical_loss": 3.341157072418721,
+      "tokens_seen": 3096772608
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2429786551115391e-05,
+      "loss": 1.235,
+      "theoretical_loss": 3.34114636174053,
+      "tokens_seen": 3096903680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2421762156957151e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.341135651642565,
+      "tokens_seen": 3097034752
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.241373776279891e-05,
+      "loss": 1.2298,
+      "theoretical_loss": 3.34112494212477,
+      "tokens_seen": 3097165824
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.240571336864067e-05,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3411142331870898,
+      "tokens_seen": 3097296896
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2397688974482428e-05,
+      "loss": 1.2624,
+      "theoretical_loss": 3.341103524829468,
+      "tokens_seen": 3097427968
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2389664580324187e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.341092817051848,
+      "tokens_seen": 3097559040
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2381640186165946e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.3410821098541748,
+      "tokens_seen": 3097690112
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2373615792007704e-05,
+      "loss": 1.2137,
+      "theoretical_loss": 3.341071403236392,
+      "tokens_seen": 3097821184
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2365591397849464e-05,
+      "loss": 1.3276,
+      "theoretical_loss": 3.341060697198444,
+      "tokens_seen": 3097952256
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2357567003691222e-05,
+      "loss": 1.2494,
+      "theoretical_loss": 3.3410499917402747,
+      "tokens_seen": 3098083328
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2349542609532982e-05,
+      "loss": 1.256,
+      "theoretical_loss": 3.3410392868618275,
+      "tokens_seen": 3098214400
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/docs_used": 1038401,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1990761756896973,
+      "objective/train/theoretical_loss": 3.3410285825630477,
+      "objective/train/tokens_used": 1468804576,
+      "theoretical_loss": 3.3410285825630477,
+      "tokens_seen": 3098345472
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.234151821537474e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3410285825630477,
+      "tokens_seen": 3098345472
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2333493821216498e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3410178788438785,
+      "tokens_seen": 3098476544
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2325469427058258e-05,
+      "loss": 1.2129,
+      "theoretical_loss": 3.341007175704264,
+      "tokens_seen": 3098607616
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2317445032900016e-05,
+      "loss": 1.2227,
+      "theoretical_loss": 3.340996473144149,
+      "tokens_seen": 3098738688
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2309420638741775e-05,
+      "loss": 1.2106,
+      "theoretical_loss": 3.340985771163477,
+      "tokens_seen": 3098869760
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2301396244583535e-05,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3409750697621923,
+      "tokens_seen": 3099000832
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2293371850425293e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.340964368940239,
+      "tokens_seen": 3099131904
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2285347456267051e-05,
+      "loss": 1.2545,
+      "theoretical_loss": 3.340953668697561,
+      "tokens_seen": 3099262976
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2277323062108811e-05,
+      "loss": 1.2413,
+      "theoretical_loss": 3.3409429690341033,
+      "tokens_seen": 3099394048
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2269298667950569e-05,
+      "loss": 1.2743,
+      "theoretical_loss": 3.340932269949809,
+      "tokens_seen": 3099525120
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2261274273792329e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.340921571444623,
+      "tokens_seen": 3099656192
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2253249879634087e-05,
+      "loss": 1.1847,
+      "theoretical_loss": 3.3409108735184896,
+      "tokens_seen": 3099787264
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2245225485475847e-05,
+      "loss": 1.1758,
+      "theoretical_loss": 3.3409001761713517,
+      "tokens_seen": 3099918336
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2237201091317605e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.340889479403155,
+      "tokens_seen": 3100049408
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2229176697159365e-05,
+      "loss": 1.2384,
+      "theoretical_loss": 3.340878783213843,
+      "tokens_seen": 3100180480
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2221152303001123e-05,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3408680876033596,
+      "tokens_seen": 3100311552
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2213127908842883e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3408573925716496,
+      "tokens_seen": 3100442624
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2205103514684642e-05,
+      "loss": 1.2473,
+      "theoretical_loss": 3.340846698118657,
+      "tokens_seen": 3100573696
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2197079120526401e-05,
+      "loss": 1.2562,
+      "theoretical_loss": 3.340836004244326,
+      "tokens_seen": 3100704768
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.218905472636816e-05,
+      "loss": 1.2775,
+      "theoretical_loss": 3.340825310948601,
+      "tokens_seen": 3100835840
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2181030332209918e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.340814618231426,
+      "tokens_seen": 3100966912
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2173005938051678e-05,
+      "loss": 1.2828,
+      "theoretical_loss": 3.340803926092745,
+      "tokens_seen": 3101097984
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2164981543893436e-05,
+      "loss": 1.1795,
+      "theoretical_loss": 3.340793234532503,
+      "tokens_seen": 3101229056
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2156957149735196e-05,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3407825435506435,
+      "tokens_seen": 3101360128
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2148932755576954e-05,
+      "loss": 1.2773,
+      "theoretical_loss": 3.340771853147111,
+      "tokens_seen": 3101491200
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2140908361418714e-05,
+      "loss": 1.2245,
+      "theoretical_loss": 3.34076116332185,
+      "tokens_seen": 3101622272
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2132883967260472e-05,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3407504740748046,
+      "tokens_seen": 3101753344
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2124859573102232e-05,
+      "loss": 1.2566,
+      "theoretical_loss": 3.3407397854059195,
+      "tokens_seen": 3101884416
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.211683517894399e-05,
+      "loss": 1.1681,
+      "theoretical_loss": 3.3407290973151387,
+      "tokens_seen": 3102015488
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.210881078478575e-05,
+      "loss": 1.2275,
+      "theoretical_loss": 3.340718409802406,
+      "tokens_seen": 3102146560
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2100786390627508e-05,
+      "loss": 1.2109,
+      "theoretical_loss": 3.3407077228676663,
+      "tokens_seen": 3102277632
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2092761996469267e-05,
+      "loss": 1.2412,
+      "theoretical_loss": 3.3406970365108637,
+      "tokens_seen": 3102408704
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2084737602311027e-05,
+      "loss": 1.2276,
+      "theoretical_loss": 3.340686350731943,
+      "tokens_seen": 3102539776
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2076713208152785e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.340675665530848,
+      "tokens_seen": 3102670848
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2068688813994545e-05,
+      "loss": 1.2123,
+      "theoretical_loss": 3.340664980907523,
+      "tokens_seen": 3102801920
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2060664419836303e-05,
+      "loss": 1.2343,
+      "theoretical_loss": 3.3406542968619126,
+      "tokens_seen": 3102932992
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2052640025678063e-05,
+      "loss": 1.27,
+      "theoretical_loss": 3.3406436133939614,
+      "tokens_seen": 3103064064
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2044615631519821e-05,
+      "loss": 1.1237,
+      "theoretical_loss": 3.340632930503613,
+      "tokens_seen": 3103195136
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2036591237361581e-05,
+      "loss": 1.2943,
+      "theoretical_loss": 3.340622248190813,
+      "tokens_seen": 3103326208
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.202856684320334e-05,
+      "loss": 1.2652,
+      "theoretical_loss": 3.3406115664555047,
+      "tokens_seen": 3103457280
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2020542449045099e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3406008852976328,
+      "tokens_seen": 3103588352
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2012518054886857e-05,
+      "loss": 1.2228,
+      "theoretical_loss": 3.3405902047171416,
+      "tokens_seen": 3103719424
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.2004493660728616e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3405795247139762,
+      "tokens_seen": 3103850496
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1996469266570375e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.34056884528808,
+      "tokens_seen": 3103981568
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1988444872412134e-05,
+      "loss": 1.3395,
+      "theoretical_loss": 3.340558166439398,
+      "tokens_seen": 3104112640
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1980420478253892e-05,
+      "loss": 1.2453,
+      "theoretical_loss": 3.340547488167875,
+      "tokens_seen": 3104243712
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.197239608409565e-05,
+      "loss": 1.2648,
+      "theoretical_loss": 3.3405368104734543,
+      "tokens_seen": 3104374784
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.196437168993741e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.3405261333560814,
+      "tokens_seen": 3104505856
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1956347295779168e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.3405154568157003,
+      "tokens_seen": 3104636928
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1948322901620928e-05,
+      "loss": 1.2474,
+      "theoretical_loss": 3.3405047808522554,
+      "tokens_seen": 3104768000
+    },
+    {
+      "epoch": 0.88,
+      "objective/train/docs_used": 1040688,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3025739192962646,
+      "objective/train/theoretical_loss": 3.3404941054656914,
+      "objective/train/tokens_used": 1475358176,
+      "theoretical_loss": 3.3404941054656914,
+      "tokens_seen": 3104899072
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1940298507462686e-05,
+      "loss": 1.2864,
+      "theoretical_loss": 3.3404941054656914,
+      "tokens_seen": 3104899072
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1932274113304446e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3404834306559525,
+      "tokens_seen": 3105030144
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1924249719146204e-05,
+      "loss": 1.2141,
+      "theoretical_loss": 3.3404727564229835,
+      "tokens_seen": 3105161216
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1916225324987964e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.340462082766729,
+      "tokens_seen": 3105292288
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1908200930829723e-05,
+      "loss": 1.184,
+      "theoretical_loss": 3.340451409687133,
+      "tokens_seen": 3105423360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.190017653667148e-05,
+      "loss": 1.2169,
+      "theoretical_loss": 3.3404407371841405,
+      "tokens_seen": 3105554432
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.189215214251324e-05,
+      "loss": 1.2008,
+      "theoretical_loss": 3.3404300652576953,
+      "tokens_seen": 3105685504
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1884127748354999e-05,
+      "loss": 1.2394,
+      "theoretical_loss": 3.3404193939077427,
+      "tokens_seen": 3105816576
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1876103354196759e-05,
+      "loss": 1.2761,
+      "theoretical_loss": 3.3404087231342268,
+      "tokens_seen": 3105947648
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1868078960038517e-05,
+      "loss": 1.2525,
+      "theoretical_loss": 3.3403980529370925,
+      "tokens_seen": 3106078720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1860054565880277e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.3403873833162843,
+      "tokens_seen": 3106209792
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1852030171722035e-05,
+      "loss": 1.3292,
+      "theoretical_loss": 3.340376714271746,
+      "tokens_seen": 3106340864
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1844005777563795e-05,
+      "loss": 1.2439,
+      "theoretical_loss": 3.3403660458034232,
+      "tokens_seen": 3106471936
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1835981383405553e-05,
+      "loss": 1.2484,
+      "theoretical_loss": 3.34035537791126,
+      "tokens_seen": 3106603008
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1827956989247313e-05,
+      "loss": 1.2475,
+      "theoretical_loss": 3.3403447105952004,
+      "tokens_seen": 3106734080
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1819932595089071e-05,
+      "loss": 1.1995,
+      "theoretical_loss": 3.34033404385519,
+      "tokens_seen": 3106865152
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.181190820093083e-05,
+      "loss": 1.2046,
+      "theoretical_loss": 3.340323377691173,
+      "tokens_seen": 3106996224
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.180388380677259e-05,
+      "loss": 1.2286,
+      "theoretical_loss": 3.3403127121030938,
+      "tokens_seen": 3107127296
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1795859412614348e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3403020470908973,
+      "tokens_seen": 3107258368
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1787835018456108e-05,
+      "loss": 1.2337,
+      "theoretical_loss": 3.340291382654528,
+      "tokens_seen": 3107389440
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1779810624297866e-05,
+      "loss": 1.2842,
+      "theoretical_loss": 3.3402807187939305,
+      "tokens_seen": 3107520512
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1771786230139626e-05,
+      "loss": 1.1828,
+      "theoretical_loss": 3.340270055509049,
+      "tokens_seen": 3107651584
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1763761835981384e-05,
+      "loss": 1.2681,
+      "theoretical_loss": 3.3402593927998296,
+      "tokens_seen": 3107782656
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1755737441823144e-05,
+      "loss": 1.191,
+      "theoretical_loss": 3.340248730666215,
+      "tokens_seen": 3107913728
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1747713047664902e-05,
+      "loss": 1.1753,
+      "theoretical_loss": 3.340238069108151,
+      "tokens_seen": 3108044800
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1739688653506662e-05,
+      "loss": 1.3174,
+      "theoretical_loss": 3.3402274081255823,
+      "tokens_seen": 3108175872
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.173166425934842e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.340216747718453,
+      "tokens_seen": 3108306944
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.172363986519018e-05,
+      "loss": 1.2592,
+      "theoretical_loss": 3.340206087886708,
+      "tokens_seen": 3108438016
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1715615471031938e-05,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3401954286302926,
+      "tokens_seen": 3108569088
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1707591076873696e-05,
+      "loss": 1.1293,
+      "theoretical_loss": 3.3401847699491505,
+      "tokens_seen": 3108700160
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1699566682715456e-05,
+      "loss": 1.2824,
+      "theoretical_loss": 3.340174111843227,
+      "tokens_seen": 3108831232
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1691542288557215e-05,
+      "loss": 1.2658,
+      "theoretical_loss": 3.3401634543124663,
+      "tokens_seen": 3108962304
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1683517894398974e-05,
+      "loss": 1.2336,
+      "theoretical_loss": 3.340152797356814,
+      "tokens_seen": 3109093376
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1675493500240733e-05,
+      "loss": 1.2086,
+      "theoretical_loss": 3.340142140976214,
+      "tokens_seen": 3109224448
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1667469106082491e-05,
+      "loss": 1.2403,
+      "theoretical_loss": 3.3401314851706116,
+      "tokens_seen": 3109355520
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.165944471192425e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.340120829939951,
+      "tokens_seen": 3109486592
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1651420317766009e-05,
+      "loss": 1.2261,
+      "theoretical_loss": 3.340110175284177,
+      "tokens_seen": 3109617664
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1643395923607767e-05,
+      "loss": 1.2418,
+      "theoretical_loss": 3.340099521203235,
+      "tokens_seen": 3109748736
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1635371529449527e-05,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3400888676970695,
+      "tokens_seen": 3109879808
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1627347135291285e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.3400782147656245,
+      "tokens_seen": 3110010880
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.1619322741133044e-05,
+      "loss": 1.3404,
+      "theoretical_loss": 3.3400675624088456,
+      "tokens_seen": 3110141952
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1611298346974803e-05,
+      "loss": 1.1963,
+      "theoretical_loss": 3.3400569106266778,
+      "tokens_seen": 3110273024
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1603273952816562e-05,
+      "loss": 1.2033,
+      "theoretical_loss": 3.340046259419065,
+      "tokens_seen": 3110404096
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1595249558658322e-05,
+      "loss": 1.288,
+      "theoretical_loss": 3.3400356087859526,
+      "tokens_seen": 3110535168
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.158722516450008e-05,
+      "loss": 1.2267,
+      "theoretical_loss": 3.3400249587272848,
+      "tokens_seen": 3110666240
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.157920077034184e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.3400143092430072,
+      "tokens_seen": 3110797312
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1571176376183598e-05,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3400036603330645,
+      "tokens_seen": 3110928384
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1563151982025358e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.339993011997401,
+      "tokens_seen": 3111059456
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1555127587867116e-05,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3399823642359623,
+      "tokens_seen": 3111190528
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1547103193708876e-05,
+      "loss": 1.2148,
+      "theoretical_loss": 3.3399717170486922,
+      "tokens_seen": 3111321600
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1042979,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3206816911697388,
+      "objective/train/theoretical_loss": 3.3399610704355363,
+      "objective/train/tokens_used": 1481911776,
+      "theoretical_loss": 3.3399610704355363,
+      "tokens_seen": 3111452672
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1539078799550634e-05,
+      "loss": 1.2609,
+      "theoretical_loss": 3.3399610704355363,
+      "tokens_seen": 3111452672
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1531054405392392e-05,
+      "loss": 1.2392,
+      "theoretical_loss": 3.3399504243964397,
+      "tokens_seen": 3111583744
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1523030011234152e-05,
+      "loss": 1.3472,
+      "theoretical_loss": 3.3399397789313467,
+      "tokens_seen": 3111714816
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.151500561707591e-05,
+      "loss": 1.1338,
+      "theoretical_loss": 3.339929134040202,
+      "tokens_seen": 3111845888
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.150698122291767e-05,
+      "loss": 1.2509,
+      "theoretical_loss": 3.339918489722951,
+      "tokens_seen": 3111976960
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1498956828759429e-05,
+      "loss": 1.3512,
+      "theoretical_loss": 3.339907845979539,
+      "tokens_seen": 3112108032
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1490932434601188e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.33989720280991,
+      "tokens_seen": 3112239104
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1482908040442947e-05,
+      "loss": 1.1957,
+      "theoretical_loss": 3.3398865602140084,
+      "tokens_seen": 3112370176
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1474883646284707e-05,
+      "loss": 1.2708,
+      "theoretical_loss": 3.339875918191781,
+      "tokens_seen": 3112501248
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1466859252126465e-05,
+      "loss": 1.2634,
+      "theoretical_loss": 3.339865276743171,
+      "tokens_seen": 3112632320
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1458834857968225e-05,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3398546358681243,
+      "tokens_seen": 3112763392
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1450810463809983e-05,
+      "loss": 1.2869,
+      "theoretical_loss": 3.3398439955665857,
+      "tokens_seen": 3112894464
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1442786069651743e-05,
+      "loss": 1.2417,
+      "theoretical_loss": 3.3398333558384996,
+      "tokens_seen": 3113025536
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1434761675493501e-05,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3398227166838117,
+      "tokens_seen": 3113156608
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.142673728133526e-05,
+      "loss": 1.2739,
+      "theoretical_loss": 3.3398120781024665,
+      "tokens_seen": 3113287680
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.141871288717702e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.339801440094409,
+      "tokens_seen": 3113418752
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1410688493018777e-05,
+      "loss": 1.2443,
+      "theoretical_loss": 3.339790802659584,
+      "tokens_seen": 3113549824
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1402664098860537e-05,
+      "loss": 1.239,
+      "theoretical_loss": 3.339780165797937,
+      "tokens_seen": 3113680896
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1394639704702296e-05,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3397695295094123,
+      "tokens_seen": 3113811968
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1386615310544055e-05,
+      "loss": 1.2908,
+      "theoretical_loss": 3.339758893793956,
+      "tokens_seen": 3113943040
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1378590916385814e-05,
+      "loss": 1.3257,
+      "theoretical_loss": 3.3397482586515115,
+      "tokens_seen": 3114074112
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1370566522227574e-05,
+      "loss": 1.1733,
+      "theoretical_loss": 3.339737624082025,
+      "tokens_seen": 3114205184
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1362542128069332e-05,
+      "loss": 1.2765,
+      "theoretical_loss": 3.339726990085442,
+      "tokens_seen": 3114336256
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1354517733911092e-05,
+      "loss": 1.2613,
+      "theoretical_loss": 3.3397163566617056,
+      "tokens_seen": 3114467328
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.134649333975285e-05,
+      "loss": 1.2917,
+      "theoretical_loss": 3.339705723810763,
+      "tokens_seen": 3114598400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1338468945594608e-05,
+      "loss": 1.2493,
+      "theoretical_loss": 3.3396950915325574,
+      "tokens_seen": 3114729472
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1330444551436368e-05,
+      "loss": 1.2536,
+      "theoretical_loss": 3.339684459827035,
+      "tokens_seen": 3114860544
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1322420157278126e-05,
+      "loss": 1.2373,
+      "theoretical_loss": 3.3396738286941403,
+      "tokens_seen": 3114991616
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1314395763119884e-05,
+      "loss": 1.2527,
+      "theoretical_loss": 3.339663198133819,
+      "tokens_seen": 3115122688
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1306371368961644e-05,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3396525681460156,
+      "tokens_seen": 3115253760
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1298346974803403e-05,
+      "loss": 1.3159,
+      "theoretical_loss": 3.3396419387306757,
+      "tokens_seen": 3115384832
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.129032258064516e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.3396313098877437,
+      "tokens_seen": 3115515904
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.128229818648692e-05,
+      "loss": 1.3097,
+      "theoretical_loss": 3.339620681617165,
+      "tokens_seen": 3115646976
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1274273792328679e-05,
+      "loss": 1.1467,
+      "theoretical_loss": 3.339610053918885,
+      "tokens_seen": 3115778048
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1266249398170439e-05,
+      "loss": 1.1716,
+      "theoretical_loss": 3.339599426792848,
+      "tokens_seen": 3115909120
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1258225004012197e-05,
+      "loss": 1.2081,
+      "theoretical_loss": 3.339588800239,
+      "tokens_seen": 3116040192
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1250200609853957e-05,
+      "loss": 1.2697,
+      "theoretical_loss": 3.339578174257286,
+      "tokens_seen": 3116171264
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1242176215695715e-05,
+      "loss": 1.2579,
+      "theoretical_loss": 3.3395675488476506,
+      "tokens_seen": 3116302336
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1234151821537473e-05,
+      "loss": 1.2321,
+      "theoretical_loss": 3.339556924010039,
+      "tokens_seen": 3116433408
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1226127427379233e-05,
+      "loss": 1.2829,
+      "theoretical_loss": 3.3395462997443977,
+      "tokens_seen": 3116564480
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1218103033220991e-05,
+      "loss": 1.2266,
+      "theoretical_loss": 3.33953567605067,
+      "tokens_seen": 3116695552
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1210078639062751e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.339525052928802,
+      "tokens_seen": 3116826624
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.120205424490451e-05,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3395144303787383,
+      "tokens_seen": 3116957696
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.119402985074627e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.339503808400425,
+      "tokens_seen": 3117088768
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1186005456588028e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.3394931869938063,
+      "tokens_seen": 3117219840
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1177981062429788e-05,
+      "loss": 1.2161,
+      "theoretical_loss": 3.3394825661588285,
+      "tokens_seen": 3117350912
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1169956668271546e-05,
+      "loss": 1.2581,
+      "theoretical_loss": 3.339471945895436,
+      "tokens_seen": 3117481984
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1161932274113306e-05,
+      "loss": 1.2547,
+      "theoretical_loss": 3.3394613262035735,
+      "tokens_seen": 3117613056
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1153907879955064e-05,
+      "loss": 1.179,
+      "theoretical_loss": 3.339450707083188,
+      "tokens_seen": 3117744128
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1145883485796822e-05,
+      "loss": 1.1555,
+      "theoretical_loss": 3.339440088534223,
+      "tokens_seen": 3117875200
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1045203,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.211982250213623,
+      "objective/train/theoretical_loss": 3.339429470556624,
+      "objective/train/tokens_used": 1488465376,
+      "theoretical_loss": 3.339429470556624,
+      "tokens_seen": 3118006272
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1137859091638582e-05,
+      "loss": 1.223,
+      "theoretical_loss": 3.339429470556624,
+      "tokens_seen": 3118006272
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.112983469748034e-05,
+      "loss": 1.2407,
+      "theoretical_loss": 3.339418853150337,
+      "tokens_seen": 3118137344
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.11218103033221e-05,
+      "loss": 1.3452,
+      "theoretical_loss": 3.3394082363153066,
+      "tokens_seen": 3118268416
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1113785909163858e-05,
+      "loss": 1.204,
+      "theoretical_loss": 3.3393976200514786,
+      "tokens_seen": 3118399488
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1105761515005618e-05,
+      "loss": 1.2289,
+      "theoretical_loss": 3.339387004358798,
+      "tokens_seen": 3118530560
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1097737120847376e-05,
+      "loss": 1.2459,
+      "theoretical_loss": 3.3393763892372097,
+      "tokens_seen": 3118661632
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1089712726689136e-05,
+      "loss": 1.2153,
+      "theoretical_loss": 3.3393657746866596,
+      "tokens_seen": 3118792704
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1081688332530895e-05,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3393551607070924,
+      "tokens_seen": 3118923776
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1073663938372654e-05,
+      "loss": 1.3042,
+      "theoretical_loss": 3.3393445472984538,
+      "tokens_seen": 3119054848
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1065639544214413e-05,
+      "loss": 1.2725,
+      "theoretical_loss": 3.339333934460689,
+      "tokens_seen": 3119185920
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1057615150056171e-05,
+      "loss": 1.2595,
+      "theoretical_loss": 3.339323322193743,
+      "tokens_seen": 3119316992
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.104959075589793e-05,
+      "loss": 1.2065,
+      "theoretical_loss": 3.3393127104975617,
+      "tokens_seen": 3119448064
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1041566361739689e-05,
+      "loss": 1.2591,
+      "theoretical_loss": 3.33930209937209,
+      "tokens_seen": 3119579136
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1033541967581449e-05,
+      "loss": 1.2285,
+      "theoretical_loss": 3.3392914888172736,
+      "tokens_seen": 3119710208
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1025517573423207e-05,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3392808788330575,
+      "tokens_seen": 3119841280
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1017493179264967e-05,
+      "loss": 1.2982,
+      "theoretical_loss": 3.3392702694193868,
+      "tokens_seen": 3119972352
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1009468785106725e-05,
+      "loss": 1.238,
+      "theoretical_loss": 3.3392596605762073,
+      "tokens_seen": 3120103424
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.1001444390948485e-05,
+      "loss": 1.2448,
+      "theoretical_loss": 3.3392490523034644,
+      "tokens_seen": 3120234496
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0993419996790243e-05,
+      "loss": 1.2031,
+      "theoretical_loss": 3.339238444601103,
+      "tokens_seen": 3120365568
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0985395602632002e-05,
+      "loss": 1.3127,
+      "theoretical_loss": 3.3392278374690694,
+      "tokens_seen": 3120496640
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0977371208473761e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.3392172309073076,
+      "tokens_seen": 3120627712
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.096934681431552e-05,
+      "loss": 1.2404,
+      "theoretical_loss": 3.3392066249157644,
+      "tokens_seen": 3120758784
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0961322420157278e-05,
+      "loss": 1.2193,
+      "theoretical_loss": 3.339196019494384,
+      "tokens_seen": 3120889856
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0953298025999038e-05,
+      "loss": 1.2542,
+      "theoretical_loss": 3.339185414643113,
+      "tokens_seen": 3121020928
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0945273631840796e-05,
+      "loss": 1.2415,
+      "theoretical_loss": 3.3391748103618957,
+      "tokens_seen": 3121152000
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0937249237682554e-05,
+      "loss": 1.2555,
+      "theoretical_loss": 3.339164206650678,
+      "tokens_seen": 3121283072
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0929224843524314e-05,
+      "loss": 1.2479,
+      "theoretical_loss": 3.339153603509405,
+      "tokens_seen": 3121414144
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0921200449366072e-05,
+      "loss": 1.3044,
+      "theoretical_loss": 3.339143000938023,
+      "tokens_seen": 3121545216
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0913176055207832e-05,
+      "loss": 1.2704,
+      "theoretical_loss": 3.339132398936477,
+      "tokens_seen": 3121676288
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.090515166104959e-05,
+      "loss": 1.3053,
+      "theoretical_loss": 3.339121797504712,
+      "tokens_seen": 3121807360
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.089712726689135e-05,
+      "loss": 1.2561,
+      "theoretical_loss": 3.3391111966426736,
+      "tokens_seen": 3121938432
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0889102872733109e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.339100596350308,
+      "tokens_seen": 3122069504
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0881078478574869e-05,
+      "loss": 1.2517,
+      "theoretical_loss": 3.3390899966275596,
+      "tokens_seen": 3122200576
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0873054084416627e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.339079397474375,
+      "tokens_seen": 3122331648
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0865029690258385e-05,
+      "loss": 1.2896,
+      "theoretical_loss": 3.3390687988906986,
+      "tokens_seen": 3122462720
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0857005296100145e-05,
+      "loss": 1.3066,
+      "theoretical_loss": 3.339058200876477,
+      "tokens_seen": 3122593792
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0848980901941903e-05,
+      "loss": 1.2253,
+      "theoretical_loss": 3.3390476034316543,
+      "tokens_seen": 3122724864
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0840956507783663e-05,
+      "loss": 1.2126,
+      "theoretical_loss": 3.339037006556177,
+      "tokens_seen": 3122855936
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0832932113625421e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.339026410249991,
+      "tokens_seen": 3122987008
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0824907719467181e-05,
+      "loss": 1.2741,
+      "theoretical_loss": 3.339015814513041,
+      "tokens_seen": 3123118080
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.081688332530894e-05,
+      "loss": 1.2783,
+      "theoretical_loss": 3.3390052193452724,
+      "tokens_seen": 3123249152
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.08088589311507e-05,
+      "loss": 1.2015,
+      "theoretical_loss": 3.3389946247466313,
+      "tokens_seen": 3123380224
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0800834536992457e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.3389840307170635,
+      "tokens_seen": 3123511296
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0792810142834217e-05,
+      "loss": 1.2559,
+      "theoretical_loss": 3.338973437256514,
+      "tokens_seen": 3123642368
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0784785748675976e-05,
+      "loss": 1.2827,
+      "theoretical_loss": 3.3389628443649277,
+      "tokens_seen": 3123773440
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0776761354517735e-05,
+      "loss": 1.2576,
+      "theoretical_loss": 3.338952252042252,
+      "tokens_seen": 3123904512
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0768736960359494e-05,
+      "loss": 1.1882,
+      "theoretical_loss": 3.3389416602884308,
+      "tokens_seen": 3124035584
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0760712566201252e-05,
+      "loss": 1.2965,
+      "theoretical_loss": 3.3389310691034106,
+      "tokens_seen": 3124166656
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0752688172043012e-05,
+      "loss": 1.2885,
+      "theoretical_loss": 3.3389204784871365,
+      "tokens_seen": 3124297728
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.074466377788477e-05,
+      "loss": 1.2762,
+      "theoretical_loss": 3.3389098884395545,
+      "tokens_seen": 3124428800
+    },
+    {
+      "epoch": 0.89,
+      "objective/train/docs_used": 1047326,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2148133516311646,
+      "objective/train/theoretical_loss": 3.33889929896061,
+      "objective/train/tokens_used": 1495018976,
+      "theoretical_loss": 3.33889929896061,
+      "tokens_seen": 3124559872
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.073663938372653e-05,
+      "loss": 1.2325,
+      "theoretical_loss": 3.33889929896061,
+      "tokens_seen": 3124559872
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0728614989568288e-05,
+      "loss": 1.2627,
+      "theoretical_loss": 3.3388887100502487,
+      "tokens_seen": 3124690944
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0720590595410048e-05,
+      "loss": 1.1917,
+      "theoretical_loss": 3.338878121708416,
+      "tokens_seen": 3124822016
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0712566201251806e-05,
+      "loss": 1.3342,
+      "theoretical_loss": 3.3388675339350575,
+      "tokens_seen": 3124953088
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0704541807093566e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.33885694673012,
+      "tokens_seen": 3125084160
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0696517412935324e-05,
+      "loss": 1.2341,
+      "theoretical_loss": 3.338846360093547,
+      "tokens_seen": 3125215232
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0688493018777084e-05,
+      "loss": 1.293,
+      "theoretical_loss": 3.338835774025286,
+      "tokens_seen": 3125346304
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0680468624618842e-05,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3388251885252815,
+      "tokens_seen": 3125477376
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.06724442304606e-05,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3388146035934803,
+      "tokens_seen": 3125608448
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.066441983630236e-05,
+      "loss": 1.2477,
+      "theoretical_loss": 3.338804019229827,
+      "tokens_seen": 3125739520
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0656395442144119e-05,
+      "loss": 1.1805,
+      "theoretical_loss": 3.3387934354342677,
+      "tokens_seen": 3125870592
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0648371047985877e-05,
+      "loss": 1.2436,
+      "theoretical_loss": 3.338782852206748,
+      "tokens_seen": 3126001664
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0640346653827637e-05,
+      "loss": 1.2107,
+      "theoretical_loss": 3.338772269547214,
+      "tokens_seen": 3126132736
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0632322259669395e-05,
+      "loss": 1.2323,
+      "theoretical_loss": 3.338761687455611,
+      "tokens_seen": 3126263808
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0624297865511153e-05,
+      "loss": 1.1991,
+      "theoretical_loss": 3.338751105931885,
+      "tokens_seen": 3126394880
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0616273471352913e-05,
+      "loss": 1.2284,
+      "theoretical_loss": 3.338740524975981,
+      "tokens_seen": 3126525952
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.0608249077194671e-05,
+      "loss": 1.2421,
+      "theoretical_loss": 3.3387299445878456,
+      "tokens_seen": 3126657024
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0600224683036431e-05,
+      "loss": 1.271,
+      "theoretical_loss": 3.3387193647674245,
+      "tokens_seen": 3126788096
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.059220028887819e-05,
+      "loss": 1.2641,
+      "theoretical_loss": 3.3387087855146627,
+      "tokens_seen": 3126919168
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0584175894719948e-05,
+      "loss": 1.2543,
+      "theoretical_loss": 3.3386982068295064,
+      "tokens_seen": 3127050240
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0576151500561708e-05,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3386876287119014,
+      "tokens_seen": 3127181312
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0568127106403466e-05,
+      "loss": 1.177,
+      "theoretical_loss": 3.3386770511617936,
+      "tokens_seen": 3127312384
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0560102712245226e-05,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3386664741791283,
+      "tokens_seen": 3127443456
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0552078318086984e-05,
+      "loss": 1.2622,
+      "theoretical_loss": 3.338655897763852,
+      "tokens_seen": 3127574528
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0544053923928744e-05,
+      "loss": 1.2928,
+      "theoretical_loss": 3.338645321915909,
+      "tokens_seen": 3127705600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0536029529770502e-05,
+      "loss": 1.236,
+      "theoretical_loss": 3.3386347466352473,
+      "tokens_seen": 3127836672
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0528005135612262e-05,
+      "loss": 1.2845,
+      "theoretical_loss": 3.3386241719218113,
+      "tokens_seen": 3127967744
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.051998074145402e-05,
+      "loss": 1.2247,
+      "theoretical_loss": 3.3386135977755464,
+      "tokens_seen": 3128098816
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.051195634729578e-05,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3386030241963995,
+      "tokens_seen": 3128229888
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0503931953137538e-05,
+      "loss": 1.2573,
+      "theoretical_loss": 3.3385924511843164,
+      "tokens_seen": 3128360960
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0495907558979298e-05,
+      "loss": 1.3054,
+      "theoretical_loss": 3.338581878739242,
+      "tokens_seen": 3128492032
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0487883164821056e-05,
+      "loss": 1.2446,
+      "theoretical_loss": 3.3385713068611227,
+      "tokens_seen": 3128623104
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0479858770662815e-05,
+      "loss": 1.2233,
+      "theoretical_loss": 3.3385607355499043,
+      "tokens_seen": 3128754176
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0471834376504575e-05,
+      "loss": 1.288,
+      "theoretical_loss": 3.338550164805533,
+      "tokens_seen": 3128885248
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0463809982346333e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.338539594627954,
+      "tokens_seen": 3129016320
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0455785588188093e-05,
+      "loss": 1.2506,
+      "theoretical_loss": 3.3385290250171136,
+      "tokens_seen": 3129147392
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0447761194029851e-05,
+      "loss": 1.1883,
+      "theoretical_loss": 3.3385184559729573,
+      "tokens_seen": 3129278464
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.043973679987161e-05,
+      "loss": 1.2301,
+      "theoretical_loss": 3.3385078874954317,
+      "tokens_seen": 3129409536
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0431712405713369e-05,
+      "loss": 1.2714,
+      "theoretical_loss": 3.338497319584482,
+      "tokens_seen": 3129540608
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0423688011555129e-05,
+      "loss": 1.2791,
+      "theoretical_loss": 3.3384867522400543,
+      "tokens_seen": 3129671680
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0415663617396887e-05,
+      "loss": 1.2713,
+      "theoretical_loss": 3.3384761854620946,
+      "tokens_seen": 3129802752
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0407639223238647e-05,
+      "loss": 1.3065,
+      "theoretical_loss": 3.338465619250549,
+      "tokens_seen": 3129933824
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0399614829080405e-05,
+      "loss": 1.2124,
+      "theoretical_loss": 3.3384550536053625,
+      "tokens_seen": 3130064896
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0391590434922163e-05,
+      "loss": 1.2895,
+      "theoretical_loss": 3.3384444885264823,
+      "tokens_seen": 3130195968
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0383566040763923e-05,
+      "loss": 1.2638,
+      "theoretical_loss": 3.3384339240138536,
+      "tokens_seen": 3130327040
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0375541646605682e-05,
+      "loss": 1.1872,
+      "theoretical_loss": 3.3384233600674227,
+      "tokens_seen": 3130458112
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0367517252447442e-05,
+      "loss": 1.1934,
+      "theoretical_loss": 3.338412796687135,
+      "tokens_seen": 3130589184
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.03594928582892e-05,
+      "loss": 1.2167,
+      "theoretical_loss": 3.338402233872937,
+      "tokens_seen": 3130720256
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.035146846413096e-05,
+      "loss": 1.1798,
+      "theoretical_loss": 3.338391671624774,
+      "tokens_seen": 3130851328
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0343444069972718e-05,
+      "loss": 1.2073,
+      "theoretical_loss": 3.338381109942593,
+      "tokens_seen": 3130982400
+    },
+    {
+      "epoch": 0.9,
+      "objective/train/docs_used": 1049585,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.266596794128418,
+      "objective/train/theoretical_loss": 3.3383705488263393,
+      "objective/train/tokens_used": 1501572576,
+      "theoretical_loss": 3.3383705488263393,
+      "tokens_seen": 3131113472
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0335419675814478e-05,
+      "loss": 1.2683,
+      "theoretical_loss": 3.3383705488263393,
+      "tokens_seen": 3131113472
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0327395281656236e-05,
+      "loss": 1.2291,
+      "theoretical_loss": 3.3383599882759585,
+      "tokens_seen": 3131244544
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0319370887497994e-05,
+      "loss": 1.2486,
+      "theoretical_loss": 3.338349428291398,
+      "tokens_seen": 3131375616
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0311346493339754e-05,
+      "loss": 1.2163,
+      "theoretical_loss": 3.338338868872602,
+      "tokens_seen": 3131506688
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0303322099181512e-05,
+      "loss": 1.2224,
+      "theoretical_loss": 3.338328310019518,
+      "tokens_seen": 3131637760
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.029529770502327e-05,
+      "loss": 1.2009,
+      "theoretical_loss": 3.3383177517320908,
+      "tokens_seen": 3131768832
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.028727331086503e-05,
+      "loss": 1.2131,
+      "theoretical_loss": 3.3383071940102678,
+      "tokens_seen": 3131899904
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0279248916706789e-05,
+      "loss": 1.1486,
+      "theoretical_loss": 3.3382966368539937,
+      "tokens_seen": 3132030976
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0271224522548547e-05,
+      "loss": 1.2516,
+      "theoretical_loss": 3.3382860802632157,
+      "tokens_seen": 3132162048
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0263200128390307e-05,
+      "loss": 1.2578,
+      "theoretical_loss": 3.3382755242378788,
+      "tokens_seen": 3132293120
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0255175734232065e-05,
+      "loss": 1.2881,
+      "theoretical_loss": 3.3382649687779296,
+      "tokens_seen": 3132424192
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0247151340073825e-05,
+      "loss": 1.2589,
+      "theoretical_loss": 3.338254413883314,
+      "tokens_seen": 3132555264
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0239126945915583e-05,
+      "loss": 1.2027,
+      "theoretical_loss": 3.338243859553978,
+      "tokens_seen": 3132686336
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0231102551757343e-05,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3382333057898683,
+      "tokens_seen": 3132817408
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0223078157599101e-05,
+      "loss": 1.2272,
+      "theoretical_loss": 3.33822275259093,
+      "tokens_seen": 3132948480
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0215053763440861e-05,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3382121999571104,
+      "tokens_seen": 3133079552
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.020702936928262e-05,
+      "loss": 1.2843,
+      "theoretical_loss": 3.3382016478883543,
+      "tokens_seen": 3133210624
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0199004975124378e-05,
+      "loss": 1.2386,
+      "theoretical_loss": 3.338191096384609,
+      "tokens_seen": 3133341696
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0190980580966137e-05,
+      "loss": 1.2789,
+      "theoretical_loss": 3.338180545445819,
+      "tokens_seen": 3133472768
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0182956186807896e-05,
+      "loss": 1.2818,
+      "theoretical_loss": 3.338169995071932,
+      "tokens_seen": 3133603840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0174931792649656e-05,
+      "loss": 1.2114,
+      "theoretical_loss": 3.3381594452628938,
+      "tokens_seen": 3133734912
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0166907398491414e-05,
+      "loss": 1.1819,
+      "theoretical_loss": 3.3381488960186503,
+      "tokens_seen": 3133865984
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0158883004333174e-05,
+      "loss": 1.2116,
+      "theoretical_loss": 3.3381383473391475,
+      "tokens_seen": 3133997056
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0150858610174932e-05,
+      "loss": 1.244,
+      "theoretical_loss": 3.3381277992243312,
+      "tokens_seen": 3134128128
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0142834216016692e-05,
+      "loss": 1.2706,
+      "theoretical_loss": 3.3381172516741486,
+      "tokens_seen": 3134259200
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.013480982185845e-05,
+      "loss": 1.2469,
+      "theoretical_loss": 3.3381067046885455,
+      "tokens_seen": 3134390272
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.012678542770021e-05,
+      "loss": 1.1582,
+      "theoretical_loss": 3.3380961582674673,
+      "tokens_seen": 3134521344
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0118761033541968e-05,
+      "loss": 1.2629,
+      "theoretical_loss": 3.338085612410861,
+      "tokens_seen": 3134652416
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0110736639383726e-05,
+      "loss": 1.199,
+      "theoretical_loss": 3.3380750671186727,
+      "tokens_seen": 3134783488
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0102712245225486e-05,
+      "loss": 1.2925,
+      "theoretical_loss": 3.3380645223908485,
+      "tokens_seen": 3134914560
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0094687851067244e-05,
+      "loss": 1.2738,
+      "theoretical_loss": 3.338053978227334,
+      "tokens_seen": 3135045632
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0086663456909004e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.338043434628077,
+      "tokens_seen": 3135176704
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0078639062750763e-05,
+      "loss": 1.243,
+      "theoretical_loss": 3.3380328915930217,
+      "tokens_seen": 3135307776
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0070614668592522e-05,
+      "loss": 1.1821,
+      "theoretical_loss": 3.3380223491221157,
+      "tokens_seen": 3135438848
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.006259027443428e-05,
+      "loss": 1.2093,
+      "theoretical_loss": 3.338011807215305,
+      "tokens_seen": 3135569920
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.005456588027604e-05,
+      "loss": 1.245,
+      "theoretical_loss": 3.3380012658725353,
+      "tokens_seen": 3135700992
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0046541486117799e-05,
+      "loss": 1.2862,
+      "theoretical_loss": 3.3379907250937535,
+      "tokens_seen": 3135832064
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0038517091959559e-05,
+      "loss": 1.2568,
+      "theoretical_loss": 3.3379801848789055,
+      "tokens_seen": 3135963136
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0030492697801317e-05,
+      "loss": 1.1982,
+      "theoretical_loss": 3.3379696452279375,
+      "tokens_seen": 3136094208
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0022468303643077e-05,
+      "loss": 1.2756,
+      "theoretical_loss": 3.3379591061407963,
+      "tokens_seen": 3136225280
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0014443909484835e-05,
+      "loss": 1.2045,
+      "theoretical_loss": 3.337948567617427,
+      "tokens_seen": 3136356352
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.0006419515326593e-05,
+      "loss": 1.2064,
+      "theoretical_loss": 3.3379380296577774,
+      "tokens_seen": 3136487424
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.998395121168353e-06,
+      "loss": 1.195,
+      "theoretical_loss": 3.3379274922617928,
+      "tokens_seen": 3136618496
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.990370727010111e-06,
+      "loss": 1.2119,
+      "theoretical_loss": 3.33791695542942,
+      "tokens_seen": 3136749568
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.982346332851871e-06,
+      "loss": 1.2071,
+      "theoretical_loss": 3.3379064191606043,
+      "tokens_seen": 3136880640
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.97432193869363e-06,
+      "loss": 1.2038,
+      "theoretical_loss": 3.3378958834552934,
+      "tokens_seen": 3137011712
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.966297544535388e-06,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3378853483134328,
+      "tokens_seen": 3137142784
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.958273150377148e-06,
+      "loss": 1.2795,
+      "theoretical_loss": 3.337874813734969,
+      "tokens_seen": 3137273856
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.950248756218906e-06,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3378642797198483,
+      "tokens_seen": 3137404928
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.942224362060664e-06,
+      "loss": 1.1946,
+      "theoretical_loss": 3.337853746268017,
+      "tokens_seen": 3137536000
+    },
+    {
+      "epoch": 0.9,
+      "objective/train/docs_used": 1051923,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1971135139465332,
+      "objective/train/theoretical_loss": 3.337843213379422,
+      "objective/train/tokens_used": 1508126176,
+      "theoretical_loss": 3.337843213379422,
+      "tokens_seen": 3137667072
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.934199967902424e-06,
+      "loss": 1.2003,
+      "theoretical_loss": 3.337843213379422,
+      "tokens_seen": 3137667072
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.926175573744182e-06,
+      "loss": 1.206,
+      "theoretical_loss": 3.337832681054009,
+      "tokens_seen": 3137798144
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.91815117958594e-06,
+      "loss": 1.2369,
+      "theoretical_loss": 3.3378221492917244,
+      "tokens_seen": 3137929216
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.9101267854277e-06,
+      "loss": 1.2811,
+      "theoretical_loss": 3.337811618092515,
+      "tokens_seen": 3138060288
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.902102391269458e-06,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3378010874563264,
+      "tokens_seen": 3138191360
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.894077997111218e-06,
+      "loss": 1.263,
+      "theoretical_loss": 3.337790557383106,
+      "tokens_seen": 3138322432
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.886053602952977e-06,
+      "loss": 1.2285,
+      "theoretical_loss": 3.3377800278727996,
+      "tokens_seen": 3138453504
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.878029208794736e-06,
+      "loss": 1.2183,
+      "theoretical_loss": 3.3377694989253532,
+      "tokens_seen": 3138584576
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.870004814636495e-06,
+      "loss": 1.265,
+      "theoretical_loss": 3.3377589705407145,
+      "tokens_seen": 3138715648
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.861980420478255e-06,
+      "loss": 1.2528,
+      "theoretical_loss": 3.3377484427188286,
+      "tokens_seen": 3138846720
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.853956026320013e-06,
+      "loss": 1.2492,
+      "theoretical_loss": 3.3377379154596425,
+      "tokens_seen": 3138977792
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.845931632161773e-06,
+      "loss": 1.2551,
+      "theoretical_loss": 3.3377273887631027,
+      "tokens_seen": 3139108864
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.837907238003531e-06,
+      "loss": 1.27,
+      "theoretical_loss": 3.337716862629155,
+      "tokens_seen": 3139239936
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.82988284384529e-06,
+      "loss": 1.2383,
+      "theoretical_loss": 3.337706337057747,
+      "tokens_seen": 3139371008
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.821858449687049e-06,
+      "loss": 1.148,
+      "theoretical_loss": 3.3376958120488243,
+      "tokens_seen": 3139502080
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.813834055528807e-06,
+      "loss": 1.2184,
+      "theoretical_loss": 3.3376852876023335,
+      "tokens_seen": 3139633152
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.805809661370567e-06,
+      "loss": 1.2208,
+      "theoretical_loss": 3.337674763718221,
+      "tokens_seen": 3139764224
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.797785267212325e-06,
+      "loss": 1.2088,
+      "theoretical_loss": 3.3376642403964336,
+      "tokens_seen": 3139895296
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.789760873054085e-06,
+      "loss": 1.279,
+      "theoretical_loss": 3.3376537176369174,
+      "tokens_seen": 3140026368
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.781736478895843e-06,
+      "loss": 1.3004,
+      "theoretical_loss": 3.337643195439619,
+      "tokens_seen": 3140157440
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.773712084737603e-06,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3376326738044853,
+      "tokens_seen": 3140288512
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.765687690579362e-06,
+      "loss": 1.2535,
+      "theoretical_loss": 3.3376221527314622,
+      "tokens_seen": 3140419584
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.757663296421122e-06,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3376116322204967,
+      "tokens_seen": 3140550656
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.74963890226288e-06,
+      "loss": 1.2251,
+      "theoretical_loss": 3.337601112271535,
+      "tokens_seen": 3140681728
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.74161450810464e-06,
+      "loss": 1.2598,
+      "theoretical_loss": 3.3375905928845238,
+      "tokens_seen": 3140812800
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.733590113946398e-06,
+      "loss": 1.2418,
+      "theoretical_loss": 3.3375800740594093,
+      "tokens_seen": 3140943872
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.725565719788156e-06,
+      "loss": 1.2432,
+      "theoretical_loss": 3.3375695557961382,
+      "tokens_seen": 3141074944
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.717541325629916e-06,
+      "loss": 1.2173,
+      "theoretical_loss": 3.337559038094658,
+      "tokens_seen": 3141206016
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.709516931471674e-06,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3375485209549134,
+      "tokens_seen": 3141337088
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.701492537313434e-06,
+      "loss": 1.2621,
+      "theoretical_loss": 3.337538004376852,
+      "tokens_seen": 3141468160
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.693468143155192e-06,
+      "loss": 1.2141,
+      "theoretical_loss": 3.337527488360421,
+      "tokens_seen": 3141599232
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.685443748996952e-06,
+      "loss": 1.211,
+      "theoretical_loss": 3.337516972905566,
+      "tokens_seen": 3141730304
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.67741935483871e-06,
+      "loss": 1.1742,
+      "theoretical_loss": 3.337506458012234,
+      "tokens_seen": 3141861376
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.66939496068047e-06,
+      "loss": 1.2814,
+      "theoretical_loss": 3.3374959436803713,
+      "tokens_seen": 3141992448
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.661370566522229e-06,
+      "loss": 1.2672,
+      "theoretical_loss": 3.3374854299099246,
+      "tokens_seen": 3142123520
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.653346172363987e-06,
+      "loss": 1.2752,
+      "theoretical_loss": 3.337474916700841,
+      "tokens_seen": 3142254592
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.645321778205747e-06,
+      "loss": 1.2518,
+      "theoretical_loss": 3.3374644040530663,
+      "tokens_seen": 3142385664
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.637297384047505e-06,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3374538919665473,
+      "tokens_seen": 3142516736
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.629272989889263e-06,
+      "loss": 1.2103,
+      "theoretical_loss": 3.3374433804412313,
+      "tokens_seen": 3142647808
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.621248595731023e-06,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3374328694770643,
+      "tokens_seen": 3142778880
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.613224201572781e-06,
+      "loss": 1.2575,
+      "theoretical_loss": 3.3374223590739933,
+      "tokens_seen": 3142909952
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.60519980741454e-06,
+      "loss": 1.2318,
+      "theoretical_loss": 3.337411849231964,
+      "tokens_seen": 3143041024
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 9.5971754132563e-06,
+      "loss": 1.2347,
+      "theoretical_loss": 3.337401339950925,
+      "tokens_seen": 3143172096
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.589151019098058e-06,
+      "loss": 1.3064,
+      "theoretical_loss": 3.337390831230821,
+      "tokens_seen": 3143303168
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.581126624939817e-06,
+      "loss": 1.3188,
+      "theoretical_loss": 3.3373803230715997,
+      "tokens_seen": 3143434240
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.573102230781576e-06,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3373698154732074,
+      "tokens_seen": 3143565312
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.565077836623336e-06,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3373593084355915,
+      "tokens_seen": 3143696384
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.557053442465094e-06,
+      "loss": 1.2245,
+      "theoretical_loss": 3.3373488019586977,
+      "tokens_seen": 3143827456
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.549029048306854e-06,
+      "loss": 1.2374,
+      "theoretical_loss": 3.3373382960424727,
+      "tokens_seen": 3143958528
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.541004654148612e-06,
+      "loss": 1.3203,
+      "theoretical_loss": 3.337327790686864,
+      "tokens_seen": 3144089600
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1054249,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.276533842086792,
+      "objective/train/theoretical_loss": 3.3373172858918183,
+      "objective/train/tokens_used": 1514679776,
+      "theoretical_loss": 3.3373172858918183,
+      "tokens_seen": 3144220672
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.53298025999037e-06,
+      "loss": 1.2899,
+      "theoretical_loss": 3.3373172858918183,
+      "tokens_seen": 3144220672
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.52495586583213e-06,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3373067816572815,
+      "tokens_seen": 3144351744
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.516931471673888e-06,
+      "loss": 1.226,
+      "theoretical_loss": 3.337296277983201,
+      "tokens_seen": 3144482816
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.508907077515648e-06,
+      "loss": 1.276,
+      "theoretical_loss": 3.3372857748695233,
+      "tokens_seen": 3144613888
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.500882683357406e-06,
+      "loss": 1.1828,
+      "theoretical_loss": 3.3372752723161954,
+      "tokens_seen": 3144744960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.492858289199166e-06,
+      "loss": 1.2436,
+      "theoretical_loss": 3.3372647703231633,
+      "tokens_seen": 3144876032
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.484833895040924e-06,
+      "loss": 1.2419,
+      "theoretical_loss": 3.3372542688903746,
+      "tokens_seen": 3145007104
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.476809500882684e-06,
+      "loss": 1.2507,
+      "theoretical_loss": 3.3372437680177756,
+      "tokens_seen": 3145138176
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.468785106724443e-06,
+      "loss": 1.2004,
+      "theoretical_loss": 3.3372332677053134,
+      "tokens_seen": 3145269248
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.460760712566202e-06,
+      "loss": 1.247,
+      "theoretical_loss": 3.3372227679529347,
+      "tokens_seen": 3145400320
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.45273631840796e-06,
+      "loss": 1.3121,
+      "theoretical_loss": 3.337212268760586,
+      "tokens_seen": 3145531392
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.444711924249719e-06,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3372017701282144,
+      "tokens_seen": 3145662464
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.436687530091479e-06,
+      "loss": 1.2484,
+      "theoretical_loss": 3.337191272055766,
+      "tokens_seen": 3145793536
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.428663135933237e-06,
+      "loss": 1.1984,
+      "theoretical_loss": 3.337180774543189,
+      "tokens_seen": 3145924608
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.420638741774997e-06,
+      "loss": 1.2361,
+      "theoretical_loss": 3.3371702775904293,
+      "tokens_seen": 3146055680
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.412614347616755e-06,
+      "loss": 1.1672,
+      "theoretical_loss": 3.3371597811974336,
+      "tokens_seen": 3146186752
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.404589953458515e-06,
+      "loss": 1.2208,
+      "theoretical_loss": 3.3371492853641493,
+      "tokens_seen": 3146317824
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.396565559300273e-06,
+      "loss": 1.2479,
+      "theoretical_loss": 3.3371387900905227,
+      "tokens_seen": 3146448896
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.388541165142033e-06,
+      "loss": 1.2189,
+      "theoretical_loss": 3.337128295376501,
+      "tokens_seen": 3146579968
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.380516770983791e-06,
+      "loss": 1.2387,
+      "theoretical_loss": 3.3371178012220306,
+      "tokens_seen": 3146711040
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.372492376825551e-06,
+      "loss": 1.2031,
+      "theoretical_loss": 3.3371073076270585,
+      "tokens_seen": 3146842112
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.36446798266731e-06,
+      "loss": 1.2329,
+      "theoretical_loss": 3.3370968145915323,
+      "tokens_seen": 3146973184
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.356443588509068e-06,
+      "loss": 1.2278,
+      "theoretical_loss": 3.337086322115398,
+      "tokens_seen": 3147104256
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.348419194350828e-06,
+      "loss": 1.2093,
+      "theoretical_loss": 3.337075830198603,
+      "tokens_seen": 3147235328
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.340394800192586e-06,
+      "loss": 1.2742,
+      "theoretical_loss": 3.337065338841094,
+      "tokens_seen": 3147366400
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.332370406034346e-06,
+      "loss": 1.2513,
+      "theoretical_loss": 3.337054848042818,
+      "tokens_seen": 3147497472
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.324346011876104e-06,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3370443578037214,
+      "tokens_seen": 3147628544
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.316321617717864e-06,
+      "loss": 1.2167,
+      "theoretical_loss": 3.3370338681237515,
+      "tokens_seen": 3147759616
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.308297223559622e-06,
+      "loss": 1.2854,
+      "theoretical_loss": 3.3370233790028556,
+      "tokens_seen": 3147890688
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.30027282940138e-06,
+      "loss": 1.2018,
+      "theoretical_loss": 3.33701289044098,
+      "tokens_seen": 3148021760
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.29224843524314e-06,
+      "loss": 1.3105,
+      "theoretical_loss": 3.3370024024380722,
+      "tokens_seen": 3148152832
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.284224041084898e-06,
+      "loss": 1.2769,
+      "theoretical_loss": 3.336991914994078,
+      "tokens_seen": 3148283904
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.276199646926657e-06,
+      "loss": 1.2556,
+      "theoretical_loss": 3.3369814281089463,
+      "tokens_seen": 3148414976
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.268175252768416e-06,
+      "loss": 1.2542,
+      "theoretical_loss": 3.336970941782622,
+      "tokens_seen": 3148546048
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.260150858610175e-06,
+      "loss": 1.1836,
+      "theoretical_loss": 3.336960456015053,
+      "tokens_seen": 3148677120
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.252126464451933e-06,
+      "loss": 1.2413,
+      "theoretical_loss": 3.336949970806187,
+      "tokens_seen": 3148808192
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.244102070293693e-06,
+      "loss": 1.2719,
+      "theoretical_loss": 3.33693948615597,
+      "tokens_seen": 3148939264
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.236077676135451e-06,
+      "loss": 1.3164,
+      "theoretical_loss": 3.3369290020643487,
+      "tokens_seen": 3149070336
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.228053281977211e-06,
+      "loss": 1.2274,
+      "theoretical_loss": 3.336918518531271,
+      "tokens_seen": 3149201408
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.220028887818969e-06,
+      "loss": 1.2354,
+      "theoretical_loss": 3.3369080355566836,
+      "tokens_seen": 3149332480
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.212004493660729e-06,
+      "loss": 1.2757,
+      "theoretical_loss": 3.3368975531405334,
+      "tokens_seen": 3149463552
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.203980099502487e-06,
+      "loss": 1.258,
+      "theoretical_loss": 3.336887071282767,
+      "tokens_seen": 3149594624
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.195955705344247e-06,
+      "loss": 1.2224,
+      "theoretical_loss": 3.336876589983332,
+      "tokens_seen": 3149725696
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.187931311186005e-06,
+      "loss": 1.2664,
+      "theoretical_loss": 3.3368661092421754,
+      "tokens_seen": 3149856768
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.179906917027765e-06,
+      "loss": 1.2891,
+      "theoretical_loss": 3.336855629059244,
+      "tokens_seen": 3149987840
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.171882522869524e-06,
+      "loss": 1.2059,
+      "theoretical_loss": 3.3368451494344855,
+      "tokens_seen": 3150118912
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.163858128711282e-06,
+      "loss": 1.2155,
+      "theoretical_loss": 3.3368346703678458,
+      "tokens_seen": 3150249984
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.155833734553042e-06,
+      "loss": 1.3008,
+      "theoretical_loss": 3.3368241918592725,
+      "tokens_seen": 3150381056
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.1478093403948e-06,
+      "loss": 1.241,
+      "theoretical_loss": 3.3368137139087133,
+      "tokens_seen": 3150512128
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.13978494623656e-06,
+      "loss": 1.2148,
+      "theoretical_loss": 3.336803236516114,
+      "tokens_seen": 3150643200
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1056328,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2618800401687622,
+      "objective/train/theoretical_loss": 3.3367927596814226,
+      "objective/train/tokens_used": 1521233376,
+      "theoretical_loss": 3.3367927596814226,
+      "tokens_seen": 3150774272
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.131760552078318e-06,
+      "loss": 1.2693,
+      "theoretical_loss": 3.3367927596814226,
+      "tokens_seen": 3150774272
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.123736157920078e-06,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3367822834045864,
+      "tokens_seen": 3150905344
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.115711763761836e-06,
+      "loss": 1.2741,
+      "theoretical_loss": 3.3367718076855515,
+      "tokens_seen": 3151036416
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.107687369603596e-06,
+      "loss": 1.1756,
+      "theoretical_loss": 3.3367613325242655,
+      "tokens_seen": 3151167488
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.099662975445354e-06,
+      "loss": 1.249,
+      "theoretical_loss": 3.3367508579206757,
+      "tokens_seen": 3151298560
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.091638581287114e-06,
+      "loss": 1.2641,
+      "theoretical_loss": 3.336740383874729,
+      "tokens_seen": 3151429632
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.083614187128872e-06,
+      "loss": 1.2415,
+      "theoretical_loss": 3.336729910386373,
+      "tokens_seen": 3151560704
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.075589792970632e-06,
+      "loss": 1.2407,
+      "theoretical_loss": 3.336719437455554,
+      "tokens_seen": 3151691776
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.06756539881239e-06,
+      "loss": 1.2489,
+      "theoretical_loss": 3.3367089650822197,
+      "tokens_seen": 3151822848
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.059541004654149e-06,
+      "loss": 1.2838,
+      "theoretical_loss": 3.336698493266317,
+      "tokens_seen": 3151953920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.051516610495909e-06,
+      "loss": 1.279,
+      "theoretical_loss": 3.336688022007793,
+      "tokens_seen": 3152084992
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.043492216337667e-06,
+      "loss": 1.2688,
+      "theoretical_loss": 3.3366775513065954,
+      "tokens_seen": 3152216064
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.035467822179427e-06,
+      "loss": 1.2481,
+      "theoretical_loss": 3.3366670811626706,
+      "tokens_seen": 3152347136
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.027443428021185e-06,
+      "loss": 1.2255,
+      "theoretical_loss": 3.3366566115759664,
+      "tokens_seen": 3152478208
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.019419033862945e-06,
+      "loss": 1.232,
+      "theoretical_loss": 3.3366461425464293,
+      "tokens_seen": 3152609280
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.011394639704703e-06,
+      "loss": 1.32,
+      "theoretical_loss": 3.3366356740740075,
+      "tokens_seen": 3152740352
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.003370245546463e-06,
+      "loss": 1.2524,
+      "theoretical_loss": 3.336625206158647,
+      "tokens_seen": 3152871424
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.995345851388221e-06,
+      "loss": 1.2582,
+      "theoretical_loss": 3.336614738800296,
+      "tokens_seen": 3153002496
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.987321457229981e-06,
+      "loss": 1.2027,
+      "theoretical_loss": 3.336604271998901,
+      "tokens_seen": 3153133568
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.97929706307174e-06,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3365938057544096,
+      "tokens_seen": 3153264640
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.971272668913497e-06,
+      "loss": 1.2011,
+      "theoretical_loss": 3.336583340066769,
+      "tokens_seen": 3153395712
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.963248274755257e-06,
+      "loss": 1.1977,
+      "theoretical_loss": 3.3365728749359267,
+      "tokens_seen": 3153526784
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.955223880597016e-06,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3365624103618288,
+      "tokens_seen": 3153657856
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.947199486438774e-06,
+      "loss": 1.2329,
+      "theoretical_loss": 3.336551946344424,
+      "tokens_seen": 3153788928
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.939175092280534e-06,
+      "loss": 1.2738,
+      "theoretical_loss": 3.336541482883659,
+      "tokens_seen": 3153920000
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.931150698122292e-06,
+      "loss": 1.2385,
+      "theoretical_loss": 3.33653101997948,
+      "tokens_seen": 3154051072
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.92312630396405e-06,
+      "loss": 1.2504,
+      "theoretical_loss": 3.3365205576318364,
+      "tokens_seen": 3154182144
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.91510190980581e-06,
+      "loss": 1.1982,
+      "theoretical_loss": 3.3365100958406737,
+      "tokens_seen": 3154313216
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.907077515647568e-06,
+      "loss": 1.1798,
+      "theoretical_loss": 3.3364996346059397,
+      "tokens_seen": 3154444288
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.899053121489328e-06,
+      "loss": 1.2071,
+      "theoretical_loss": 3.336489173927582,
+      "tokens_seen": 3154575360
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.891028727331086e-06,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3364787138055476,
+      "tokens_seen": 3154706432
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.883004333172845e-06,
+      "loss": 1.2987,
+      "theoretical_loss": 3.3364682542397834,
+      "tokens_seen": 3154837504
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.874979939014604e-06,
+      "loss": 1.1974,
+      "theoretical_loss": 3.336457795230238,
+      "tokens_seen": 3154968576
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.866955544856363e-06,
+      "loss": 1.2777,
+      "theoretical_loss": 3.3364473367768572,
+      "tokens_seen": 3155099648
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.858931150698123e-06,
+      "loss": 1.2377,
+      "theoretical_loss": 3.3364368788795895,
+      "tokens_seen": 3155230720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.85090675653988e-06,
+      "loss": 1.2921,
+      "theoretical_loss": 3.3364264215383814,
+      "tokens_seen": 3155361792
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.84288236238164e-06,
+      "loss": 1.305,
+      "theoretical_loss": 3.33641596475318,
+      "tokens_seen": 3155492864
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.834857968223399e-06,
+      "loss": 1.2699,
+      "theoretical_loss": 3.336405508523934,
+      "tokens_seen": 3155623936
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.826833574065159e-06,
+      "loss": 1.2114,
+      "theoretical_loss": 3.33639505285059,
+      "tokens_seen": 3155755008
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.818809179906917e-06,
+      "loss": 1.3163,
+      "theoretical_loss": 3.3363845977330953,
+      "tokens_seen": 3155886080
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.810784785748677e-06,
+      "loss": 1.2247,
+      "theoretical_loss": 3.336374143171397,
+      "tokens_seen": 3156017152
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.802760391590435e-06,
+      "loss": 1.2235,
+      "theoretical_loss": 3.336363689165443,
+      "tokens_seen": 3156148224
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.794735997432195e-06,
+      "loss": 1.2333,
+      "theoretical_loss": 3.33635323571518,
+      "tokens_seen": 3156279296
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.786711603273953e-06,
+      "loss": 1.2893,
+      "theoretical_loss": 3.336342782820556,
+      "tokens_seen": 3156410368
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.778687209115711e-06,
+      "loss": 1.2644,
+      "theoretical_loss": 3.3363323304815187,
+      "tokens_seen": 3156541440
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.770662814957471e-06,
+      "loss": 1.3021,
+      "theoretical_loss": 3.3363218786980147,
+      "tokens_seen": 3156672512
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.76263842079923e-06,
+      "loss": 1.2073,
+      "theoretical_loss": 3.3363114274699917,
+      "tokens_seen": 3156803584
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.75461402664099e-06,
+      "loss": 1.2684,
+      "theoretical_loss": 3.3363009767973972,
+      "tokens_seen": 3156934656
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.746589632482748e-06,
+      "loss": 1.2249,
+      "theoretical_loss": 3.3362905266801786,
+      "tokens_seen": 3157065728
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.738565238324508e-06,
+      "loss": 1.2567,
+      "theoretical_loss": 3.336280077118283,
+      "tokens_seen": 3157196800
+    },
+    {
+      "epoch": 0.91,
+      "objective/train/docs_used": 1058478,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.228834867477417,
+      "objective/train/theoretical_loss": 3.3362696281116584,
+      "objective/train/tokens_used": 1527786976,
+      "theoretical_loss": 3.3362696281116584,
+      "tokens_seen": 3157327872
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.730540844166266e-06,
+      "loss": 1.2279,
+      "theoretical_loss": 3.3362696281116584,
+      "tokens_seen": 3157327872
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.722516450008026e-06,
+      "loss": 1.2424,
+      "theoretical_loss": 3.3362591796602517,
+      "tokens_seen": 3157458944
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.714492055849784e-06,
+      "loss": 1.2612,
+      "theoretical_loss": 3.336248731764011,
+      "tokens_seen": 3157590016
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.706467661691544e-06,
+      "loss": 1.2086,
+      "theoretical_loss": 3.3362382844228833,
+      "tokens_seen": 3157721088
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.698443267533302e-06,
+      "loss": 1.2709,
+      "theoretical_loss": 3.336227837636816,
+      "tokens_seen": 3157852160
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.69041887337506e-06,
+      "loss": 1.3042,
+      "theoretical_loss": 3.3362173914057567,
+      "tokens_seen": 3157983232
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.68239447921682e-06,
+      "loss": 1.2812,
+      "theoretical_loss": 3.336206945729653,
+      "tokens_seen": 3158114304
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.674370085058578e-06,
+      "loss": 1.216,
+      "theoretical_loss": 3.3361965006084526,
+      "tokens_seen": 3158245376
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.666345690900338e-06,
+      "loss": 1.2713,
+      "theoretical_loss": 3.336186056042102,
+      "tokens_seen": 3158376448
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.658321296742097e-06,
+      "loss": 1.2543,
+      "theoretical_loss": 3.33617561203055,
+      "tokens_seen": 3158507520
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.650296902583856e-06,
+      "loss": 1.3341,
+      "theoretical_loss": 3.3361651685737437,
+      "tokens_seen": 3158638592
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.642272508425615e-06,
+      "loss": 1.2034,
+      "theoretical_loss": 3.33615472567163,
+      "tokens_seen": 3158769664
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.634248114267373e-06,
+      "loss": 1.2456,
+      "theoretical_loss": 3.3361442833241566,
+      "tokens_seen": 3158900736
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.626223720109133e-06,
+      "loss": 1.2276,
+      "theoretical_loss": 3.3361338415312716,
+      "tokens_seen": 3159031808
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.618199325950891e-06,
+      "loss": 1.308,
+      "theoretical_loss": 3.336123400292922,
+      "tokens_seen": 3159162880
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.61017493179265e-06,
+      "loss": 1.2425,
+      "theoretical_loss": 3.336112959609056,
+      "tokens_seen": 3159293952
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.602150537634409e-06,
+      "loss": 1.1767,
+      "theoretical_loss": 3.3361025194796206,
+      "tokens_seen": 3159425024
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.594126143476167e-06,
+      "loss": 1.2585,
+      "theoretical_loss": 3.336092079904563,
+      "tokens_seen": 3159556096
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 8.586101749317925e-06,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3360816408838314,
+      "tokens_seen": 3159687168
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.578077355159685e-06,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3360712024173735,
+      "tokens_seen": 3159818240
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.570052961001444e-06,
+      "loss": 1.2089,
+      "theoretical_loss": 3.3360607645051363,
+      "tokens_seen": 3159949312
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.562028566843204e-06,
+      "loss": 1.1761,
+      "theoretical_loss": 3.336050327147068,
+      "tokens_seen": 3160080384
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.554004172684962e-06,
+      "loss": 1.2609,
+      "theoretical_loss": 3.3360398903431157,
+      "tokens_seen": 3160211456
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.545979778526722e-06,
+      "loss": 1.2457,
+      "theoretical_loss": 3.336029454093227,
+      "tokens_seen": 3160342528
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.53795538436848e-06,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3360190183973497,
+      "tokens_seen": 3160473600
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.52993099021024e-06,
+      "loss": 1.3125,
+      "theoretical_loss": 3.3360085832554316,
+      "tokens_seen": 3160604672
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.521906596051998e-06,
+      "loss": 1.1833,
+      "theoretical_loss": 3.3359981486674197,
+      "tokens_seen": 3160735744
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.513882201893758e-06,
+      "loss": 1.1992,
+      "theoretical_loss": 3.3359877146332626,
+      "tokens_seen": 3160866816
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.505857807735516e-06,
+      "loss": 1.2321,
+      "theoretical_loss": 3.335977281152907,
+      "tokens_seen": 3160997888
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.497833413577274e-06,
+      "loss": 1.2673,
+      "theoretical_loss": 3.335966848226301,
+      "tokens_seen": 3161128960
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.489809019419034e-06,
+      "loss": 1.2873,
+      "theoretical_loss": 3.3359564158533925,
+      "tokens_seen": 3161260032
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.481784625260792e-06,
+      "loss": 1.2069,
+      "theoretical_loss": 3.3359459840341286,
+      "tokens_seen": 3161391104
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.473760231102552e-06,
+      "loss": 1.1908,
+      "theoretical_loss": 3.335935552768457,
+      "tokens_seen": 3161522176
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.46573583694431e-06,
+      "loss": 1.2457,
+      "theoretical_loss": 3.3359251220563255,
+      "tokens_seen": 3161653248
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.45771144278607e-06,
+      "loss": 1.2413,
+      "theoretical_loss": 3.335914691897682,
+      "tokens_seen": 3161784320
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.449687048627829e-06,
+      "loss": 1.1986,
+      "theoretical_loss": 3.3359042622924746,
+      "tokens_seen": 3161915392
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.441662654469589e-06,
+      "loss": 1.2325,
+      "theoretical_loss": 3.33589383324065,
+      "tokens_seen": 3162046464
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.433638260311347e-06,
+      "loss": 1.1991,
+      "theoretical_loss": 3.335883404742156,
+      "tokens_seen": 3162177536
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.425613866153107e-06,
+      "loss": 1.2934,
+      "theoretical_loss": 3.3358729767969413,
+      "tokens_seen": 3162308608
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.417589471994865e-06,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3358625494049523,
+      "tokens_seen": 3162439680
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.409565077836623e-06,
+      "loss": 1.271,
+      "theoretical_loss": 3.3358521225661377,
+      "tokens_seen": 3162570752
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.401540683678383e-06,
+      "loss": 1.2656,
+      "theoretical_loss": 3.335841696280445,
+      "tokens_seen": 3162701824
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.393516289520141e-06,
+      "loss": 1.1877,
+      "theoretical_loss": 3.335831270547822,
+      "tokens_seen": 3162832896
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.385491895361901e-06,
+      "loss": 1.1994,
+      "theoretical_loss": 3.335820845368216,
+      "tokens_seen": 3162963968
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.37746750120366e-06,
+      "loss": 1.2697,
+      "theoretical_loss": 3.335810420741575,
+      "tokens_seen": 3163095040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.36944310704542e-06,
+      "loss": 1.3017,
+      "theoretical_loss": 3.3357999966678467,
+      "tokens_seen": 3163226112
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.361418712887177e-06,
+      "loss": 1.2683,
+      "theoretical_loss": 3.3357895731469793,
+      "tokens_seen": 3163357184
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.353394318728937e-06,
+      "loss": 1.2537,
+      "theoretical_loss": 3.3357791501789205,
+      "tokens_seen": 3163488256
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.345369924570696e-06,
+      "loss": 1.23,
+      "theoretical_loss": 3.335768727763617,
+      "tokens_seen": 3163619328
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.337345530412455e-06,
+      "loss": 1.2344,
+      "theoretical_loss": 3.335758305901018,
+      "tokens_seen": 3163750400
+    },
+    {
+      "epoch": 0.92,
+      "objective/train/docs_used": 1060550,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3011760711669922,
+      "objective/train/theoretical_loss": 3.3357478845910706,
+      "objective/train/tokens_used": 1534340576,
+      "theoretical_loss": 3.3357478845910706,
+      "tokens_seen": 3163881472
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.329321136254214e-06,
+      "loss": 1.2742,
+      "theoretical_loss": 3.3357478845910706,
+      "tokens_seen": 3163881472
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.321296742095974e-06,
+      "loss": 1.2688,
+      "theoretical_loss": 3.3357374638337225,
+      "tokens_seen": 3164012544
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.313272347937732e-06,
+      "loss": 1.2469,
+      "theoretical_loss": 3.335727043628922,
+      "tokens_seen": 3164143616
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.30524795377949e-06,
+      "loss": 1.2375,
+      "theoretical_loss": 3.3357166239766167,
+      "tokens_seen": 3164274688
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.29722355962125e-06,
+      "loss": 1.2693,
+      "theoretical_loss": 3.335706204876754,
+      "tokens_seen": 3164405760
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.289199165463008e-06,
+      "loss": 1.2883,
+      "theoretical_loss": 3.335695786329282,
+      "tokens_seen": 3164536832
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.281174771304766e-06,
+      "loss": 1.2582,
+      "theoretical_loss": 3.335685368334149,
+      "tokens_seen": 3164667904
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.273150377146526e-06,
+      "loss": 1.2173,
+      "theoretical_loss": 3.335674950891302,
+      "tokens_seen": 3164798976
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.265125982988284e-06,
+      "loss": 1.192,
+      "theoretical_loss": 3.33566453400069,
+      "tokens_seen": 3164930048
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.257101588830043e-06,
+      "loss": 1.2829,
+      "theoretical_loss": 3.3356541176622594,
+      "tokens_seen": 3165061120
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.249077194671803e-06,
+      "loss": 1.2659,
+      "theoretical_loss": 3.3356437018759593,
+      "tokens_seen": 3165192192
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.24105280051356e-06,
+      "loss": 1.2687,
+      "theoretical_loss": 3.335633286641737,
+      "tokens_seen": 3165323264
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.23302840635532e-06,
+      "loss": 1.2055,
+      "theoretical_loss": 3.3356228719595404,
+      "tokens_seen": 3165454336
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.225004012197079e-06,
+      "loss": 1.2696,
+      "theoretical_loss": 3.335612457829318,
+      "tokens_seen": 3165585408
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.216979618038837e-06,
+      "loss": 1.2029,
+      "theoretical_loss": 3.3356020442510164,
+      "tokens_seen": 3165716480
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.208955223880597e-06,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3355916312245846,
+      "tokens_seen": 3165847552
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.200930829722355e-06,
+      "loss": 1.1989,
+      "theoretical_loss": 3.33558121874997,
+      "tokens_seen": 3165978624
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.192906435564115e-06,
+      "loss": 1.1992,
+      "theoretical_loss": 3.335570806827121,
+      "tokens_seen": 3166109696
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.184882041405873e-06,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3355603954559845,
+      "tokens_seen": 3166240768
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.176857647247633e-06,
+      "loss": 1.2877,
+      "theoretical_loss": 3.3355499846365095,
+      "tokens_seen": 3166371840
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.168833253089391e-06,
+      "loss": 1.2159,
+      "theoretical_loss": 3.335539574368644,
+      "tokens_seen": 3166502912
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.160808858931151e-06,
+      "loss": 1.3175,
+      "theoretical_loss": 3.335529164652335,
+      "tokens_seen": 3166633984
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.15278446477291e-06,
+      "loss": 1.2174,
+      "theoretical_loss": 3.335518755487531,
+      "tokens_seen": 3166765056
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.14476007061467e-06,
+      "loss": 1.3045,
+      "theoretical_loss": 3.33550834687418,
+      "tokens_seen": 3166896128
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.136735676456428e-06,
+      "loss": 1.2184,
+      "theoretical_loss": 3.33549793881223,
+      "tokens_seen": 3167027200
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.128711282298188e-06,
+      "loss": 1.2985,
+      "theoretical_loss": 3.335487531301629,
+      "tokens_seen": 3167158272
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.120686888139946e-06,
+      "loss": 1.3238,
+      "theoretical_loss": 3.3354771243423245,
+      "tokens_seen": 3167289344
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.112662493981704e-06,
+      "loss": 1.2217,
+      "theoretical_loss": 3.3354667179342647,
+      "tokens_seen": 3167420416
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.104638099823464e-06,
+      "loss": 1.2277,
+      "theoretical_loss": 3.335456312077398,
+      "tokens_seen": 3167551488
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.096613705665222e-06,
+      "loss": 1.2662,
+      "theoretical_loss": 3.335445906771672,
+      "tokens_seen": 3167682560
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.088589311506982e-06,
+      "loss": 1.2791,
+      "theoretical_loss": 3.3354355020170345,
+      "tokens_seen": 3167813632
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.08056491734874e-06,
+      "loss": 1.2157,
+      "theoretical_loss": 3.335425097813434,
+      "tokens_seen": 3167944704
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.0725405231905e-06,
+      "loss": 1.3091,
+      "theoretical_loss": 3.3354146941608187,
+      "tokens_seen": 3168075776
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.064516129032258e-06,
+      "loss": 1.229,
+      "theoretical_loss": 3.335404291059136,
+      "tokens_seen": 3168206848
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.056491734874018e-06,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3353938885083343,
+      "tokens_seen": 3168337920
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.048467340715777e-06,
+      "loss": 1.252,
+      "theoretical_loss": 3.335383486508361,
+      "tokens_seen": 3168468992
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.040442946557536e-06,
+      "loss": 1.276,
+      "theoretical_loss": 3.335373085059165,
+      "tokens_seen": 3168600064
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.032418552399295e-06,
+      "loss": 1.2296,
+      "theoretical_loss": 3.3353626841606943,
+      "tokens_seen": 3168731136
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.024394158241053e-06,
+      "loss": 1.2855,
+      "theoretical_loss": 3.3353522838128966,
+      "tokens_seen": 3168862208
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.016369764082813e-06,
+      "loss": 1.2562,
+      "theoretical_loss": 3.33534188401572,
+      "tokens_seen": 3168993280
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.008345369924571e-06,
+      "loss": 1.267,
+      "theoretical_loss": 3.335331484769113,
+      "tokens_seen": 3169124352
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.000320975766331e-06,
+      "loss": 1.2323,
+      "theoretical_loss": 3.335321086073023,
+      "tokens_seen": 3169255424
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.992296581608089e-06,
+      "loss": 1.3118,
+      "theoretical_loss": 3.3353106879273984,
+      "tokens_seen": 3169386496
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.984272187449849e-06,
+      "loss": 1.2289,
+      "theoretical_loss": 3.335300290332187,
+      "tokens_seen": 3169517568
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.976247793291607e-06,
+      "loss": 1.2742,
+      "theoretical_loss": 3.335289893287338,
+      "tokens_seen": 3169648640
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.968223399133367e-06,
+      "loss": 1.2443,
+      "theoretical_loss": 3.335279496792798,
+      "tokens_seen": 3169779712
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.960199004975125e-06,
+      "loss": 1.3223,
+      "theoretical_loss": 3.335269100848516,
+      "tokens_seen": 3169910784
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.952174610816884e-06,
+      "loss": 1.2239,
+      "theoretical_loss": 3.3352587054544403,
+      "tokens_seen": 3170041856
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.944150216658643e-06,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3352483106105186,
+      "tokens_seen": 3170172928
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.936125822500402e-06,
+      "loss": 1.2332,
+      "theoretical_loss": 3.335237916316699,
+      "tokens_seen": 3170304000
+    },
+    {
+      "epoch": 0.92,
+      "objective/train/docs_used": 1062678,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2017204761505127,
+      "objective/train/theoretical_loss": 3.33522752257293,
+      "objective/train/tokens_used": 1540894176,
+      "theoretical_loss": 3.33522752257293,
+      "tokens_seen": 3170435072
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.92810142834216e-06,
+      "loss": 1.2221,
+      "theoretical_loss": 3.33522752257293,
+      "tokens_seen": 3170435072
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.92007703418392e-06,
+      "loss": 1.2964,
+      "theoretical_loss": 3.335217129379159,
+      "tokens_seen": 3170566144
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.912052640025678e-06,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3352067367353353,
+      "tokens_seen": 3170697216
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.904028245867436e-06,
+      "loss": 1.309,
+      "theoretical_loss": 3.335196344641407,
+      "tokens_seen": 3170828288
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.896003851709196e-06,
+      "loss": 1.2286,
+      "theoretical_loss": 3.3351859530973207,
+      "tokens_seen": 3170959360
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.887979457550954e-06,
+      "loss": 1.2222,
+      "theoretical_loss": 3.335175562103026,
+      "tokens_seen": 3171090432
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.879955063392714e-06,
+      "loss": 1.241,
+      "theoretical_loss": 3.335165171658471,
+      "tokens_seen": 3171221504
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.871930669234472e-06,
+      "loss": 1.2121,
+      "theoretical_loss": 3.3351547817636034,
+      "tokens_seen": 3171352576
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.863906275076232e-06,
+      "loss": 1.2658,
+      "theoretical_loss": 3.3351443924183712,
+      "tokens_seen": 3171483648
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.85588188091799e-06,
+      "loss": 1.2649,
+      "theoretical_loss": 3.3351340036227235,
+      "tokens_seen": 3171614720
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.84785748675975e-06,
+      "loss": 1.2268,
+      "theoretical_loss": 3.335123615376608,
+      "tokens_seen": 3171745792
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.839833092601509e-06,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3351132276799733,
+      "tokens_seen": 3171876864
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.831808698443267e-06,
+      "loss": 1.3171,
+      "theoretical_loss": 3.335102840532767,
+      "tokens_seen": 3172007936
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.823784304285027e-06,
+      "loss": 1.3023,
+      "theoretical_loss": 3.3350924539349376,
+      "tokens_seen": 3172139008
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.815759910126785e-06,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3350820678864337,
+      "tokens_seen": 3172270080
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.807735515968545e-06,
+      "loss": 1.1948,
+      "theoretical_loss": 3.335071682387203,
+      "tokens_seen": 3172401152
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.799711121810303e-06,
+      "loss": 1.2801,
+      "theoretical_loss": 3.3350612974371936,
+      "tokens_seen": 3172532224
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.791686727652063e-06,
+      "loss": 1.2944,
+      "theoretical_loss": 3.335050913036355,
+      "tokens_seen": 3172663296
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.783662333493821e-06,
+      "loss": 1.2315,
+      "theoretical_loss": 3.335040529184634,
+      "tokens_seen": 3172794368
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.775637939335581e-06,
+      "loss": 1.2152,
+      "theoretical_loss": 3.33503014588198,
+      "tokens_seen": 3172925440
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.76761354517734e-06,
+      "loss": 1.3247,
+      "theoretical_loss": 3.3350197631283405,
+      "tokens_seen": 3173056512
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.7595891510191e-06,
+      "loss": 1.229,
+      "theoretical_loss": 3.335009380923664,
+      "tokens_seen": 3173187584
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.751564756860857e-06,
+      "loss": 1.2877,
+      "theoretical_loss": 3.3349989992678992,
+      "tokens_seen": 3173318656
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.743540362702616e-06,
+      "loss": 1.2752,
+      "theoretical_loss": 3.3349886181609936,
+      "tokens_seen": 3173449728
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.735515968544376e-06,
+      "loss": 1.2079,
+      "theoretical_loss": 3.334978237602897,
+      "tokens_seen": 3173580800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.727491574386134e-06,
+      "loss": 1.2618,
+      "theoretical_loss": 3.334967857593556,
+      "tokens_seen": 3173711872
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.719467180227894e-06,
+      "loss": 1.2342,
+      "theoretical_loss": 3.3349574781329197,
+      "tokens_seen": 3173842944
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.711442786069652e-06,
+      "loss": 1.1867,
+      "theoretical_loss": 3.3349470992209365,
+      "tokens_seen": 3173974016
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.703418391911412e-06,
+      "loss": 1.2316,
+      "theoretical_loss": 3.334936720857555,
+      "tokens_seen": 3174105088
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.69539399775317e-06,
+      "loss": 1.2902,
+      "theoretical_loss": 3.3349263430427225,
+      "tokens_seen": 3174236160
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.68736960359493e-06,
+      "loss": 1.2495,
+      "theoretical_loss": 3.3349159657763887,
+      "tokens_seen": 3174367232
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.679345209436688e-06,
+      "loss": 1.2863,
+      "theoretical_loss": 3.334905589058501,
+      "tokens_seen": 3174498304
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.671320815278448e-06,
+      "loss": 1.2385,
+      "theoretical_loss": 3.334895212889008,
+      "tokens_seen": 3174629376
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.663296421120206e-06,
+      "loss": 1.1697,
+      "theoretical_loss": 3.334884837267859,
+      "tokens_seen": 3174760448
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.655272026961964e-06,
+      "loss": 1.3242,
+      "theoretical_loss": 3.3348744621950006,
+      "tokens_seen": 3174891520
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.647247632803724e-06,
+      "loss": 1.2697,
+      "theoretical_loss": 3.3348640876703826,
+      "tokens_seen": 3175022592
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.639223238645483e-06,
+      "loss": 1.2731,
+      "theoretical_loss": 3.3348537136939527,
+      "tokens_seen": 3175153664
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.631198844487242e-06,
+      "loss": 1.2296,
+      "theoretical_loss": 3.33484334026566,
+      "tokens_seen": 3175284736
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.623174450329e-06,
+      "loss": 1.3139,
+      "theoretical_loss": 3.334832967385452,
+      "tokens_seen": 3175415808
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.61515005617076e-06,
+      "loss": 1.286,
+      "theoretical_loss": 3.3348225950532777,
+      "tokens_seen": 3175546880
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.607125662012518e-06,
+      "loss": 1.2633,
+      "theoretical_loss": 3.3348122232690858,
+      "tokens_seen": 3175677952
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.599101267854278e-06,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3348018520328244,
+      "tokens_seen": 3175809024
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.591076873696036e-06,
+      "loss": 1.2202,
+      "theoretical_loss": 3.3347914813444413,
+      "tokens_seen": 3175940096
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.583052479537796e-06,
+      "loss": 1.1658,
+      "theoretical_loss": 3.334781111203886,
+      "tokens_seen": 3176071168
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.575028085379554e-06,
+      "loss": 1.1751,
+      "theoretical_loss": 3.3347707416111065,
+      "tokens_seen": 3176202240
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.567003691221314e-06,
+      "loss": 1.2387,
+      "theoretical_loss": 3.334760372566051,
+      "tokens_seen": 3176333312
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.558979297063072e-06,
+      "loss": 1.2692,
+      "theoretical_loss": 3.334750004068668,
+      "tokens_seen": 3176464384
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.5509549029048305e-06,
+      "loss": 1.2559,
+      "theoretical_loss": 3.334739636118907,
+      "tokens_seen": 3176595456
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.5429305087465905e-06,
+      "loss": 1.1716,
+      "theoretical_loss": 3.334729268716715,
+      "tokens_seen": 3176726528
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.534906114588349e-06,
+      "loss": 1.281,
+      "theoretical_loss": 3.3347189018620416,
+      "tokens_seen": 3176857600
+    },
+    {
+      "epoch": 0.93,
+      "objective/train/docs_used": 1064723,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.245157241821289,
+      "objective/train/theoretical_loss": 3.3347085355548347,
+      "objective/train/tokens_used": 1547447776,
+      "theoretical_loss": 3.3347085355548347,
+      "tokens_seen": 3176988672
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.526881720430108e-06,
+      "loss": 1.2351,
+      "theoretical_loss": 3.3347085355548347,
+      "tokens_seen": 3176988672
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.518857326271867e-06,
+      "loss": 1.2242,
+      "theoretical_loss": 3.334698169795043,
+      "tokens_seen": 3177119744
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.510832932113626e-06,
+      "loss": 1.2598,
+      "theoretical_loss": 3.334687804582615,
+      "tokens_seen": 3177250816
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.502808537955384e-06,
+      "loss": 1.2726,
+      "theoretical_loss": 3.334677439917499,
+      "tokens_seen": 3177381888
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.494784143797144e-06,
+      "loss": 1.2164,
+      "theoretical_loss": 3.334667075799644,
+      "tokens_seen": 3177512960
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.486759749638902e-06,
+      "loss": 1.222,
+      "theoretical_loss": 3.3346567122289983,
+      "tokens_seen": 3177644032
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.478735355480662e-06,
+      "loss": 1.2095,
+      "theoretical_loss": 3.33464634920551,
+      "tokens_seen": 3177775104
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.47071096132242e-06,
+      "loss": 1.2775,
+      "theoretical_loss": 3.3346359867291286,
+      "tokens_seen": 3177906176
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.4626865671641785e-06,
+      "loss": 1.2726,
+      "theoretical_loss": 3.334625624799802,
+      "tokens_seen": 3178037248
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.454662173005938e-06,
+      "loss": 1.2101,
+      "theoretical_loss": 3.3346152634174784,
+      "tokens_seen": 3178168320
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.446637778847697e-06,
+      "loss": 1.2521,
+      "theoretical_loss": 3.3346049025821074,
+      "tokens_seen": 3178299392
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.4386133846894565e-06,
+      "loss": 1.2271,
+      "theoretical_loss": 3.3345945422936367,
+      "tokens_seen": 3178430464
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.430588990531215e-06,
+      "loss": 1.2293,
+      "theoretical_loss": 3.3345841825520153,
+      "tokens_seen": 3178561536
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.422564596372975e-06,
+      "loss": 1.2616,
+      "theoretical_loss": 3.3345738233571915,
+      "tokens_seen": 3178692608
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.414540202214733e-06,
+      "loss": 1.2235,
+      "theoretical_loss": 3.334563464709114,
+      "tokens_seen": 3178823680
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.406515808056493e-06,
+      "loss": 1.1803,
+      "theoretical_loss": 3.334553106607732,
+      "tokens_seen": 3178954752
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.398491413898251e-06,
+      "loss": 1.2277,
+      "theoretical_loss": 3.334542749052993,
+      "tokens_seen": 3179085824
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.390467019740011e-06,
+      "loss": 1.2433,
+      "theoretical_loss": 3.3345323920448466,
+      "tokens_seen": 3179216896
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.382442625581769e-06,
+      "loss": 1.2728,
+      "theoretical_loss": 3.3345220355832406,
+      "tokens_seen": 3179347968
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.374418231423528e-06,
+      "loss": 1.2337,
+      "theoretical_loss": 3.3345116796681245,
+      "tokens_seen": 3179479040
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.366393837265287e-06,
+      "loss": 1.2328,
+      "theoretical_loss": 3.334501324299446,
+      "tokens_seen": 3179610112
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.358369443107045e-06,
+      "loss": 1.1953,
+      "theoretical_loss": 3.3344909694771543,
+      "tokens_seen": 3179741184
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.3503450489488045e-06,
+      "loss": 1.2286,
+      "theoretical_loss": 3.334480615201198,
+      "tokens_seen": 3179872256
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.3423206547905635e-06,
+      "loss": 1.2608,
+      "theoretical_loss": 3.3344702614715263,
+      "tokens_seen": 3180003328
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.334296260632323e-06,
+      "loss": 1.2292,
+      "theoretical_loss": 3.334459908288087,
+      "tokens_seen": 3180134400
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.326271866474081e-06,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3344495556508287,
+      "tokens_seen": 3180265472
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.318247472315841e-06,
+      "loss": 1.2986,
+      "theoretical_loss": 3.3344392035597004,
+      "tokens_seen": 3180396544
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.310223078157599e-06,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3344288520146512,
+      "tokens_seen": 3180527616
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.302198683999359e-06,
+      "loss": 1.2755,
+      "theoretical_loss": 3.3344185010156293,
+      "tokens_seen": 3180658688
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.294174289841117e-06,
+      "loss": 1.2282,
+      "theoretical_loss": 3.3344081505625836,
+      "tokens_seen": 3180789760
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.286149895682877e-06,
+      "loss": 1.2429,
+      "theoretical_loss": 3.3343978006554624,
+      "tokens_seen": 3180920832
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.278125501524635e-06,
+      "loss": 1.2861,
+      "theoretical_loss": 3.334387451294215,
+      "tokens_seen": 3181051904
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.270101107366393e-06,
+      "loss": 1.3074,
+      "theoretical_loss": 3.33437710247879,
+      "tokens_seen": 3181182976
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.262076713208153e-06,
+      "loss": 1.2707,
+      "theoretical_loss": 3.3343667542091353,
+      "tokens_seen": 3181314048
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.2540523190499115e-06,
+      "loss": 1.3261,
+      "theoretical_loss": 3.3343564064852007,
+      "tokens_seen": 3181445120
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.246027924891671e-06,
+      "loss": 1.1704,
+      "theoretical_loss": 3.3343460593069345,
+      "tokens_seen": 3181576192
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.23800353073343e-06,
+      "loss": 1.149,
+      "theoretical_loss": 3.334335712674286,
+      "tokens_seen": 3181707264
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.2299791365751895e-06,
+      "loss": 1.228,
+      "theoretical_loss": 3.334325366587203,
+      "tokens_seen": 3181838336
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.221954742416948e-06,
+      "loss": 1.2061,
+      "theoretical_loss": 3.3343150210456347,
+      "tokens_seen": 3181969408
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.213930348258708e-06,
+      "loss": 1.2427,
+      "theoretical_loss": 3.3343046760495296,
+      "tokens_seen": 3182100480
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.205905954100466e-06,
+      "loss": 1.2321,
+      "theoretical_loss": 3.334294331598837,
+      "tokens_seen": 3182231552
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.197881559942225e-06,
+      "loss": 1.2136,
+      "theoretical_loss": 3.3342839876935058,
+      "tokens_seen": 3182362624
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.189857165783984e-06,
+      "loss": 1.2708,
+      "theoretical_loss": 3.334273644333484,
+      "tokens_seen": 3182493696
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.181832771625742e-06,
+      "loss": 1.2329,
+      "theoretical_loss": 3.334263301518721,
+      "tokens_seen": 3182624768
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.173808377467501e-06,
+      "loss": 1.291,
+      "theoretical_loss": 3.3342529592491656,
+      "tokens_seen": 3182755840
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.16578398330926e-06,
+      "loss": 1.2593,
+      "theoretical_loss": 3.334242617524766,
+      "tokens_seen": 3182886912
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.157759589151019e-06,
+      "loss": 1.2501,
+      "theoretical_loss": 3.334232276345472,
+      "tokens_seen": 3183017984
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.1497351949927776e-06,
+      "loss": 1.2291,
+      "theoretical_loss": 3.3342219357112315,
+      "tokens_seen": 3183149056
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.1417108008345375e-06,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3342115956219933,
+      "tokens_seen": 3183280128
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.133686406676296e-06,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3342012560777072,
+      "tokens_seen": 3183411200
+    },
+    {
+      "epoch": 0.93,
+      "objective/train/docs_used": 1067002,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2128909826278687,
+      "objective/train/theoretical_loss": 3.3341909170783213,
+      "objective/train/tokens_used": 1554001376,
+      "theoretical_loss": 3.3341909170783213,
+      "tokens_seen": 3183542272
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.125662012518056e-06,
+      "loss": 1.1618,
+      "theoretical_loss": 3.3341909170783213,
+      "tokens_seen": 3183542272
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.117637618359814e-06,
+      "loss": 1.264,
+      "theoretical_loss": 3.334180578623785,
+      "tokens_seen": 3183673344
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.109613224201574e-06,
+      "loss": 1.2516,
+      "theoretical_loss": 3.3341702407140463,
+      "tokens_seen": 3183804416
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.101588830043332e-06,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3341599033490548,
+      "tokens_seen": 3183935488
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.093564435885092e-06,
+      "loss": 1.2288,
+      "theoretical_loss": 3.334149566528759,
+      "tokens_seen": 3184066560
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.08554004172685e-06,
+      "loss": 1.3322,
+      "theoretical_loss": 3.3341392302531077,
+      "tokens_seen": 3184197632
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.077515647568608e-06,
+      "loss": 1.2647,
+      "theoretical_loss": 3.3341288945220504,
+      "tokens_seen": 3184328704
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.069491253410368e-06,
+      "loss": 1.2409,
+      "theoretical_loss": 3.3341185593355354,
+      "tokens_seen": 3184459776
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.061466859252126e-06,
+      "loss": 1.1903,
+      "theoretical_loss": 3.334108224693512,
+      "tokens_seen": 3184590848
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.053442465093886e-06,
+      "loss": 1.2214,
+      "theoretical_loss": 3.3340978905959284,
+      "tokens_seen": 3184721920
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.0454180709356445e-06,
+      "loss": 1.3093,
+      "theoretical_loss": 3.334087557042734,
+      "tokens_seen": 3184852992
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.037393676777404e-06,
+      "loss": 1.2841,
+      "theoretical_loss": 3.3340772240338783,
+      "tokens_seen": 3184984064
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.029369282619163e-06,
+      "loss": 1.2195,
+      "theoretical_loss": 3.334066891569309,
+      "tokens_seen": 3185115136
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.021344888460922e-06,
+      "loss": 1.2696,
+      "theoretical_loss": 3.3340565596489764,
+      "tokens_seen": 3185246208
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.013320494302681e-06,
+      "loss": 1.2662,
+      "theoretical_loss": 3.334046228272828,
+      "tokens_seen": 3185377280
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.00529610014444e-06,
+      "loss": 1.1617,
+      "theoretical_loss": 3.3340358974408137,
+      "tokens_seen": 3185508352
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.997271705986198e-06,
+      "loss": 1.2272,
+      "theoretical_loss": 3.3340255671528825,
+      "tokens_seen": 3185639424
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.989247311827957e-06,
+      "loss": 1.279,
+      "theoretical_loss": 3.3340152374089826,
+      "tokens_seen": 3185770496
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.981222917669716e-06,
+      "loss": 1.2316,
+      "theoretical_loss": 3.3340049082090637,
+      "tokens_seen": 3185901568
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.973198523511474e-06,
+      "loss": 1.2197,
+      "theoretical_loss": 3.3339945795530745,
+      "tokens_seen": 3186032640
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.965174129353234e-06,
+      "loss": 1.2504,
+      "theoretical_loss": 3.333984251440964,
+      "tokens_seen": 3186163712
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.9571497351949924e-06,
+      "loss": 1.2067,
+      "theoretical_loss": 3.3339739238726813,
+      "tokens_seen": 3186294784
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.949125341036752e-06,
+      "loss": 1.162,
+      "theoretical_loss": 3.3339635968481756,
+      "tokens_seen": 3186425856
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.9411009468785106e-06,
+      "loss": 1.1877,
+      "theoretical_loss": 3.333953270367395,
+      "tokens_seen": 3186556928
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.9330765527202705e-06,
+      "loss": 1.2192,
+      "theoretical_loss": 3.3339429444302895,
+      "tokens_seen": 3186688000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.925052158562029e-06,
+      "loss": 1.2319,
+      "theoretical_loss": 3.3339326190368075,
+      "tokens_seen": 3186819072
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.917027764403789e-06,
+      "loss": 1.2294,
+      "theoretical_loss": 3.3339222941868982,
+      "tokens_seen": 3186950144
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.909003370245547e-06,
+      "loss": 1.1607,
+      "theoretical_loss": 3.3339119698805106,
+      "tokens_seen": 3187081216
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.900978976087307e-06,
+      "loss": 1.3126,
+      "theoretical_loss": 3.333901646117594,
+      "tokens_seen": 3187212288
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.892954581929065e-06,
+      "loss": 1.1777,
+      "theoretical_loss": 3.333891322898097,
+      "tokens_seen": 3187343360
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.884930187770823e-06,
+      "loss": 1.1967,
+      "theoretical_loss": 3.333881000221969,
+      "tokens_seen": 3187474432
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.876905793612583e-06,
+      "loss": 1.229,
+      "theoretical_loss": 3.333870678089159,
+      "tokens_seen": 3187605504
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.868881399454341e-06,
+      "loss": 1.277,
+      "theoretical_loss": 3.3338603564996165,
+      "tokens_seen": 3187736576
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.860857005296101e-06,
+      "loss": 1.2827,
+      "theoretical_loss": 3.3338500354532896,
+      "tokens_seen": 3187867648
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.852832611137859e-06,
+      "loss": 1.2505,
+      "theoretical_loss": 3.3338397149501278,
+      "tokens_seen": 3187998720
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.844808216979618e-06,
+      "loss": 1.185,
+      "theoretical_loss": 3.3338293949900804,
+      "tokens_seen": 3188129792
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.8367838228213775e-06,
+      "loss": 1.2614,
+      "theoretical_loss": 3.3338190755730963,
+      "tokens_seen": 3188260864
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.8287594286631365e-06,
+      "loss": 1.2657,
+      "theoretical_loss": 3.3338087566991246,
+      "tokens_seen": 3188391936
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.820735034504895e-06,
+      "loss": 1.277,
+      "theoretical_loss": 3.3337984383681145,
+      "tokens_seen": 3188523008
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.812710640346655e-06,
+      "loss": 1.1896,
+      "theoretical_loss": 3.333788120580015,
+      "tokens_seen": 3188654080
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.804686246188413e-06,
+      "loss": 1.2701,
+      "theoretical_loss": 3.3337778033347756,
+      "tokens_seen": 3188785152
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.796661852030171e-06,
+      "loss": 1.1769,
+      "theoretical_loss": 3.333767486632345,
+      "tokens_seen": 3188916224
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.788637457871931e-06,
+      "loss": 1.237,
+      "theoretical_loss": 3.3337571704726723,
+      "tokens_seen": 3189047296
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.780613063713689e-06,
+      "loss": 1.1759,
+      "theoretical_loss": 3.3337468548557068,
+      "tokens_seen": 3189178368
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.772588669555449e-06,
+      "loss": 1.2186,
+      "theoretical_loss": 3.3337365397813974,
+      "tokens_seen": 3189309440
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.764564275397207e-06,
+      "loss": 1.2387,
+      "theoretical_loss": 3.333726225249694,
+      "tokens_seen": 3189440512
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.756539881238967e-06,
+      "loss": 1.2575,
+      "theoretical_loss": 3.3337159112605446,
+      "tokens_seen": 3189571584
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.7485154870807254e-06,
+      "loss": 1.3595,
+      "theoretical_loss": 3.3337055978138994,
+      "tokens_seen": 3189702656
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.740491092922485e-06,
+      "loss": 1.1883,
+      "theoretical_loss": 3.333695284909707,
+      "tokens_seen": 3189833728
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.7324666987642435e-06,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3336849725479167,
+      "tokens_seen": 3189964800
+    },
+    {
+      "epoch": 0.93,
+      "objective/train/docs_used": 1069349,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2649257183074951,
+      "objective/train/theoretical_loss": 3.3336746607284784,
+      "objective/train/tokens_used": 1560554976,
+      "theoretical_loss": 3.3336746607284784,
+      "tokens_seen": 3190095872
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.7244423046060035e-06,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3336746607284784,
+      "tokens_seen": 3190095872
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.716417910447762e-06,
+      "loss": 1.1431,
+      "theoretical_loss": 3.33366434945134,
+      "tokens_seen": 3190226944
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.70839351628952e-06,
+      "loss": 1.301,
+      "theoretical_loss": 3.3336540387164515,
+      "tokens_seen": 3190358016
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.70036912213128e-06,
+      "loss": 1.1903,
+      "theoretical_loss": 3.333643728523762,
+      "tokens_seen": 3190489088
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.692344727973038e-06,
+      "loss": 1.2524,
+      "theoretical_loss": 3.333633418873221,
+      "tokens_seen": 3190620160
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.684320333814798e-06,
+      "loss": 1.2353,
+      "theoretical_loss": 3.3336231097647766,
+      "tokens_seen": 3190751232
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.676295939656556e-06,
+      "loss": 1.1955,
+      "theoretical_loss": 3.333612801198379,
+      "tokens_seen": 3190882304
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.668271545498315e-06,
+      "loss": 1.2185,
+      "theoretical_loss": 3.333602493173978,
+      "tokens_seen": 3191013376
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.660247151340074e-06,
+      "loss": 1.3211,
+      "theoretical_loss": 3.3335921856915216,
+      "tokens_seen": 3191144448
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.652222757181833e-06,
+      "loss": 1.201,
+      "theoretical_loss": 3.3335818787509597,
+      "tokens_seen": 3191275520
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.6441983630235915e-06,
+      "loss": 1.3245,
+      "theoretical_loss": 3.333571572352241,
+      "tokens_seen": 3191406592
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.636173968865351e-06,
+      "loss": 1.2089,
+      "theoretical_loss": 3.3335612664953156,
+      "tokens_seen": 3191537664
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.62814957470711e-06,
+      "loss": 1.2372,
+      "theoretical_loss": 3.333550961180132,
+      "tokens_seen": 3191668736
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.6201251805488695e-06,
+      "loss": 1.237,
+      "theoretical_loss": 3.33354065640664,
+      "tokens_seen": 3191799808
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.612100786390628e-06,
+      "loss": 1.2808,
+      "theoretical_loss": 3.3335303521747885,
+      "tokens_seen": 3191930880
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.604076392232386e-06,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3335200484845275,
+      "tokens_seen": 3192061952
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.596051998074146e-06,
+      "loss": 1.2603,
+      "theoretical_loss": 3.3335097453358054,
+      "tokens_seen": 3192193024
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.588027603915904e-06,
+      "loss": 1.1623,
+      "theoretical_loss": 3.333499442728572,
+      "tokens_seen": 3192324096
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.580003209757664e-06,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3334891406627762,
+      "tokens_seen": 3192455168
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.571978815599422e-06,
+      "loss": 1.2444,
+      "theoretical_loss": 3.333478839138368,
+      "tokens_seen": 3192586240
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.563954421441182e-06,
+      "loss": 1.2199,
+      "theoretical_loss": 3.333468538155296,
+      "tokens_seen": 3192717312
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.55593002728294e-06,
+      "loss": 1.309,
+      "theoretical_loss": 3.3334582377135105,
+      "tokens_seen": 3192848384
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.5479056331247e-06,
+      "loss": 1.2451,
+      "theoretical_loss": 3.3334479378129593,
+      "tokens_seen": 3192979456
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.5398812389664584e-06,
+      "loss": 1.2177,
+      "theoretical_loss": 3.3334376384535935,
+      "tokens_seen": 3193110528
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.531856844808218e-06,
+      "loss": 1.2252,
+      "theoretical_loss": 3.333427339635361,
+      "tokens_seen": 3193241600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.5238324506499765e-06,
+      "loss": 1.2554,
+      "theoretical_loss": 3.3334170413582123,
+      "tokens_seen": 3193372672
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.515808056491735e-06,
+      "loss": 1.3081,
+      "theoretical_loss": 3.3334067436220955,
+      "tokens_seen": 3193503744
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.507783662333494e-06,
+      "loss": 1.2726,
+      "theoretical_loss": 3.3333964464269616,
+      "tokens_seen": 3193634816
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.499759268175253e-06,
+      "loss": 1.2963,
+      "theoretical_loss": 3.3333861497727586,
+      "tokens_seen": 3193765888
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.491734874017012e-06,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3333758536594362,
+      "tokens_seen": 3193896960
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.48371047985877e-06,
+      "loss": 1.2038,
+      "theoretical_loss": 3.3333655580869443,
+      "tokens_seen": 3194028032
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.47568608570053e-06,
+      "loss": 1.1835,
+      "theoretical_loss": 3.333355263055232,
+      "tokens_seen": 3194159104
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.467661691542288e-06,
+      "loss": 1.222,
+      "theoretical_loss": 3.3333449685642482,
+      "tokens_seen": 3194290176
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.459637297384048e-06,
+      "loss": 1.3211,
+      "theoretical_loss": 3.333334674613943,
+      "tokens_seen": 3194421248
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.451612903225806e-06,
+      "loss": 1.2265,
+      "theoretical_loss": 3.3333243812042657,
+      "tokens_seen": 3194552320
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.443588509067566e-06,
+      "loss": 1.2588,
+      "theoretical_loss": 3.3333140883351655,
+      "tokens_seen": 3194683392
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.4355641149093245e-06,
+      "loss": 1.2571,
+      "theoretical_loss": 3.333303796006592,
+      "tokens_seen": 3194814464
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.427539720751084e-06,
+      "loss": 1.2137,
+      "theoretical_loss": 3.333293504218495,
+      "tokens_seen": 3194945536
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.419515326592843e-06,
+      "loss": 1.1574,
+      "theoretical_loss": 3.333283212970823,
+      "tokens_seen": 3195076608
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.411490932434601e-06,
+      "loss": 1.3549,
+      "theoretical_loss": 3.333272922263526,
+      "tokens_seen": 3195207680
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.403466538276361e-06,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3332626320965537,
+      "tokens_seen": 3195338752
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.395442144118119e-06,
+      "loss": 1.2147,
+      "theoretical_loss": 3.3332523424698555,
+      "tokens_seen": 3195469824
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.387417749959879e-06,
+      "loss": 1.2462,
+      "theoretical_loss": 3.33324205338338,
+      "tokens_seen": 3195600896
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.379393355801637e-06,
+      "loss": 1.2145,
+      "theoretical_loss": 3.333231764837078,
+      "tokens_seen": 3195731968
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.371368961643397e-06,
+      "loss": 1.2709,
+      "theoretical_loss": 3.3332214768308983,
+      "tokens_seen": 3195863040
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.363344567485155e-06,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3332111893647904,
+      "tokens_seen": 3195994112
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.355320173326914e-06,
+      "loss": 1.254,
+      "theoretical_loss": 3.333200902438704,
+      "tokens_seen": 3196125184
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.347295779168673e-06,
+      "loss": 1.1629,
+      "theoretical_loss": 3.333190616052588,
+      "tokens_seen": 3196256256
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.339271385010432e-06,
+      "loss": 1.2551,
+      "theoretical_loss": 3.333180330206393,
+      "tokens_seen": 3196387328
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.3312469908521906e-06,
+      "loss": 1.1697,
+      "theoretical_loss": 3.3331700449000676,
+      "tokens_seen": 3196518400
+    },
+    {
+      "epoch": 0.94,
+      "objective/train/docs_used": 1071513,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2150182723999023,
+      "objective/train/theoretical_loss": 3.3331597601335616,
+      "objective/train/tokens_used": 1567108576,
+      "theoretical_loss": 3.3331597601335616,
+      "tokens_seen": 3196649472
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.32322259669395e-06,
+      "loss": 1.2413,
+      "theoretical_loss": 3.3331597601335616,
+      "tokens_seen": 3196649472
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.315198202535709e-06,
+      "loss": 1.2104,
+      "theoretical_loss": 3.3331494759068248,
+      "tokens_seen": 3196780544
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.307173808377467e-06,
+      "loss": 1.1697,
+      "theoretical_loss": 3.3331391922198064,
+      "tokens_seen": 3196911616
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.299149414219227e-06,
+      "loss": 1.2485,
+      "theoretical_loss": 3.333128909072456,
+      "tokens_seen": 3197042688
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.291125020060985e-06,
+      "loss": 1.2429,
+      "theoretical_loss": 3.3331186264647235,
+      "tokens_seen": 3197173760
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.283100625902745e-06,
+      "loss": 1.198,
+      "theoretical_loss": 3.3331083443965577,
+      "tokens_seen": 3197304832
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.275076231744503e-06,
+      "loss": 1.237,
+      "theoretical_loss": 3.3330980628679088,
+      "tokens_seen": 3197435904
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.267051837586263e-06,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3330877818787266,
+      "tokens_seen": 3197566976
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.259027443428021e-06,
+      "loss": 1.187,
+      "theoretical_loss": 3.33307750142896,
+      "tokens_seen": 3197698048
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.251003049269781e-06,
+      "loss": 1.2305,
+      "theoretical_loss": 3.333067221518559,
+      "tokens_seen": 3197829120
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.242978655111539e-06,
+      "loss": 1.262,
+      "theoretical_loss": 3.333056942147473,
+      "tokens_seen": 3197960192
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.2349542609532984e-06,
+      "loss": 1.2018,
+      "theoretical_loss": 3.3330466633156517,
+      "tokens_seen": 3198091264
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.2269298667950575e-06,
+      "loss": 1.23,
+      "theoretical_loss": 3.3330363850230444,
+      "tokens_seen": 3198222336
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.2189054726368165e-06,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3330261072696015,
+      "tokens_seen": 3198353408
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.210881078478576e-06,
+      "loss": 1.2173,
+      "theoretical_loss": 3.333015830055272,
+      "tokens_seen": 3198484480
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.202856684320335e-06,
+      "loss": 1.2192,
+      "theoretical_loss": 3.333005553380006,
+      "tokens_seen": 3198615552
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.194832290162094e-06,
+      "loss": 1.2692,
+      "theoretical_loss": 3.3329952772437528,
+      "tokens_seen": 3198746624
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.186807896003852e-06,
+      "loss": 1.2212,
+      "theoretical_loss": 3.3329850016464615,
+      "tokens_seen": 3198877696
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.178783501845611e-06,
+      "loss": 1.1548,
+      "theoretical_loss": 3.3329747265880827,
+      "tokens_seen": 3199008768
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.17075910768737e-06,
+      "loss": 1.233,
+      "theoretical_loss": 3.3329644520685657,
+      "tokens_seen": 3199139840
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.162734713529129e-06,
+      "loss": 1.2319,
+      "theoretical_loss": 3.33295417808786,
+      "tokens_seen": 3199270912
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.154710319370887e-06,
+      "loss": 1.2577,
+      "theoretical_loss": 3.3329439046459153,
+      "tokens_seen": 3199401984
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.146685925212646e-06,
+      "loss": 1.2446,
+      "theoretical_loss": 3.332933631742682,
+      "tokens_seen": 3199533056
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.1386615310544054e-06,
+      "loss": 1.2115,
+      "theoretical_loss": 3.3329233593781082,
+      "tokens_seen": 3199664128
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.1306371368961645e-06,
+      "loss": 1.1378,
+      "theoretical_loss": 3.332913087552145,
+      "tokens_seen": 3199795200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.1226127427379236e-06,
+      "loss": 1.2162,
+      "theoretical_loss": 3.332902816264742,
+      "tokens_seen": 3199926272
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.114588348579683e-06,
+      "loss": 1.2629,
+      "theoretical_loss": 3.3328925455158482,
+      "tokens_seen": 3200057344
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.106563954421442e-06,
+      "loss": 1.1817,
+      "theoretical_loss": 3.3328822753054137,
+      "tokens_seen": 3200188416
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.098539560263201e-06,
+      "loss": 1.235,
+      "theoretical_loss": 3.3328720056333885,
+      "tokens_seen": 3200319488
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.090515166104959e-06,
+      "loss": 1.3004,
+      "theoretical_loss": 3.3328617364997215,
+      "tokens_seen": 3200450560
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.082490771946718e-06,
+      "loss": 1.2123,
+      "theoretical_loss": 3.3328514679043635,
+      "tokens_seen": 3200581632
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.074466377788477e-06,
+      "loss": 1.1661,
+      "theoretical_loss": 3.3328411998472633,
+      "tokens_seen": 3200712704
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.066441983630236e-06,
+      "loss": 1.1697,
+      "theoretical_loss": 3.3328309323283714,
+      "tokens_seen": 3200843776
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.058417589471995e-06,
+      "loss": 1.2058,
+      "theoretical_loss": 3.332820665347637,
+      "tokens_seen": 3200974848
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.050393195313754e-06,
+      "loss": 1.2252,
+      "theoretical_loss": 3.3328103989050097,
+      "tokens_seen": 3201105920
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.042368801155513e-06,
+      "loss": 1.2585,
+      "theoretical_loss": 3.3328001330004398,
+      "tokens_seen": 3201236992
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.034344406997272e-06,
+      "loss": 1.2673,
+      "theoretical_loss": 3.332789867633877,
+      "tokens_seen": 3201368064
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.026320012839031e-06,
+      "loss": 1.262,
+      "theoretical_loss": 3.332779602805271,
+      "tokens_seen": 3201499136
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.0182956186807905e-06,
+      "loss": 1.1738,
+      "theoretical_loss": 3.3327693385145714,
+      "tokens_seen": 3201630208
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.0102712245225495e-06,
+      "loss": 1.1977,
+      "theoretical_loss": 3.332759074761728,
+      "tokens_seen": 3201761280
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.002246830364308e-06,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3327488115466912,
+      "tokens_seen": 3201892352
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.994222436206067e-06,
+      "loss": 1.2182,
+      "theoretical_loss": 3.3327385488694103,
+      "tokens_seen": 3202023424
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.986198042047825e-06,
+      "loss": 1.2277,
+      "theoretical_loss": 3.332728286729835,
+      "tokens_seen": 3202154496
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.978173647889584e-06,
+      "loss": 1.2606,
+      "theoretical_loss": 3.332718025127915,
+      "tokens_seen": 3202285568
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.970149253731343e-06,
+      "loss": 1.2451,
+      "theoretical_loss": 3.332707764063601,
+      "tokens_seen": 3202416640
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.962124859573102e-06,
+      "loss": 1.2196,
+      "theoretical_loss": 3.3326975035368416,
+      "tokens_seen": 3202547712
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.954100465414861e-06,
+      "loss": 1.3492,
+      "theoretical_loss": 3.3326872435475874,
+      "tokens_seen": 3202678784
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.94607607125662e-06,
+      "loss": 1.3131,
+      "theoretical_loss": 3.3326769840957886,
+      "tokens_seen": 3202809856
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.938051677098379e-06,
+      "loss": 1.2144,
+      "theoretical_loss": 3.332666725181394,
+      "tokens_seen": 3202940928
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.9300272829401384e-06,
+      "loss": 1.2699,
+      "theoretical_loss": 3.3326564668043543,
+      "tokens_seen": 3203072000
+    },
+    {
+      "epoch": 0.94,
+      "objective/train/docs_used": 1073646,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.30216646194458,
+      "objective/train/theoretical_loss": 3.3326462089646194,
+      "objective/train/tokens_used": 1573662176,
+      "theoretical_loss": 3.3326462089646194,
+      "tokens_seen": 3203203072
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.9220028887818975e-06,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3326462089646194,
+      "tokens_seen": 3203203072
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.9139784946236566e-06,
+      "loss": 1.2222,
+      "theoretical_loss": 3.332635951662138,
+      "tokens_seen": 3203334144
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.905954100465415e-06,
+      "loss": 1.189,
+      "theoretical_loss": 3.3326256948968616,
+      "tokens_seen": 3203465216
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.897929706307174e-06,
+      "loss": 1.2597,
+      "theoretical_loss": 3.332615438668739,
+      "tokens_seen": 3203596288
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.889905312148933e-06,
+      "loss": 1.2567,
+      "theoretical_loss": 3.332605182977721,
+      "tokens_seen": 3203727360
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.881880917990692e-06,
+      "loss": 1.2593,
+      "theoretical_loss": 3.3325949278237563,
+      "tokens_seen": 3203858432
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.873856523832451e-06,
+      "loss": 1.2511,
+      "theoretical_loss": 3.332584673206796,
+      "tokens_seen": 3203989504
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.86583212967421e-06,
+      "loss": 1.1554,
+      "theoretical_loss": 3.332574419126789,
+      "tokens_seen": 3204120576
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.857807735515969e-06,
+      "loss": 1.2403,
+      "theoretical_loss": 3.332564165583686,
+      "tokens_seen": 3204251648
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.849783341357728e-06,
+      "loss": 1.2833,
+      "theoretical_loss": 3.332553912577436,
+      "tokens_seen": 3204382720
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.841758947199487e-06,
+      "loss": 1.2093,
+      "theoretical_loss": 3.33254366010799,
+      "tokens_seen": 3204513792
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.8337345530412454e-06,
+      "loss": 1.2484,
+      "theoretical_loss": 3.3325334081752973,
+      "tokens_seen": 3204644864
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.8257101588830045e-06,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3325231567793083,
+      "tokens_seen": 3204775936
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.8176857647247636e-06,
+      "loss": 1.1926,
+      "theoretical_loss": 3.3325129059199723,
+      "tokens_seen": 3204907008
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.809661370566522e-06,
+      "loss": 1.2866,
+      "theoretical_loss": 3.33250265559724,
+      "tokens_seen": 3205038080
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.801636976408281e-06,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3324924058110605,
+      "tokens_seen": 3205169152
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.79361258225004e-06,
+      "loss": 1.2666,
+      "theoretical_loss": 3.332482156561385,
+      "tokens_seen": 3205300224
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.785588188091799e-06,
+      "loss": 1.2411,
+      "theoretical_loss": 3.3324719078481624,
+      "tokens_seen": 3205431296
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.777563793933558e-06,
+      "loss": 1.1817,
+      "theoretical_loss": 3.332461659671343,
+      "tokens_seen": 3205562368
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.769539399775317e-06,
+      "loss": 1.2018,
+      "theoretical_loss": 3.332451412030877,
+      "tokens_seen": 3205693440
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.761515005617076e-06,
+      "loss": 1.1775,
+      "theoretical_loss": 3.332441164926714,
+      "tokens_seen": 3205824512
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.753490611458835e-06,
+      "loss": 1.2832,
+      "theoretical_loss": 3.3324309183588046,
+      "tokens_seen": 3205955584
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.745466217300594e-06,
+      "loss": 1.1957,
+      "theoretical_loss": 3.332420672327098,
+      "tokens_seen": 3206086656
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.737441823142353e-06,
+      "loss": 1.221,
+      "theoretical_loss": 3.332410426831545,
+      "tokens_seen": 3206217728
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.729417428984112e-06,
+      "loss": 1.261,
+      "theoretical_loss": 3.332400181872095,
+      "tokens_seen": 3206348800
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.7213930348258714e-06,
+      "loss": 1.27,
+      "theoretical_loss": 3.3323899374486987,
+      "tokens_seen": 3206479872
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.71336864066763e-06,
+      "loss": 1.231,
+      "theoretical_loss": 3.3323796935613057,
+      "tokens_seen": 3206610944
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.705344246509389e-06,
+      "loss": 1.277,
+      "theoretical_loss": 3.332369450209866,
+      "tokens_seen": 3206742016
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.697319852351148e-06,
+      "loss": 1.3021,
+      "theoretical_loss": 3.33235920739433,
+      "tokens_seen": 3206873088
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.689295458192907e-06,
+      "loss": 1.2227,
+      "theoretical_loss": 3.3323489651146474,
+      "tokens_seen": 3207004160
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.681271064034666e-06,
+      "loss": 1.2007,
+      "theoretical_loss": 3.3323387233707686,
+      "tokens_seen": 3207135232
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.673246669876425e-06,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3323284821626435,
+      "tokens_seen": 3207266304
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.665222275718184e-06,
+      "loss": 1.283,
+      "theoretical_loss": 3.332318241490222,
+      "tokens_seen": 3207397376
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.657197881559942e-06,
+      "loss": 1.1734,
+      "theoretical_loss": 3.3323080013534545,
+      "tokens_seen": 3207528448
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.649173487401701e-06,
+      "loss": 1.2643,
+      "theoretical_loss": 3.3322977617522906,
+      "tokens_seen": 3207659520
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.64114909324346e-06,
+      "loss": 1.2438,
+      "theoretical_loss": 3.332287522686681,
+      "tokens_seen": 3207790592
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.633124699085219e-06,
+      "loss": 1.2711,
+      "theoretical_loss": 3.332277284156576,
+      "tokens_seen": 3207921664
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.6251003049269784e-06,
+      "loss": 1.2957,
+      "theoretical_loss": 3.332267046161925,
+      "tokens_seen": 3208052736
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.617075910768737e-06,
+      "loss": 1.1773,
+      "theoretical_loss": 3.332256808702678,
+      "tokens_seen": 3208183808
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.609051516610496e-06,
+      "loss": 1.2716,
+      "theoretical_loss": 3.3322465717787857,
+      "tokens_seen": 3208314880
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.601027122452255e-06,
+      "loss": 1.235,
+      "theoretical_loss": 3.3322363353901983,
+      "tokens_seen": 3208445952
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.593002728294014e-06,
+      "loss": 1.1874,
+      "theoretical_loss": 3.3322260995368653,
+      "tokens_seen": 3208577024
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.584978334135773e-06,
+      "loss": 1.2491,
+      "theoretical_loss": 3.332215864218738,
+      "tokens_seen": 3208708096
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.576953939977532e-06,
+      "loss": 1.2014,
+      "theoretical_loss": 3.332205629435765,
+      "tokens_seen": 3208839168
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.568929545819291e-06,
+      "loss": 1.2373,
+      "theoretical_loss": 3.332195395187898,
+      "tokens_seen": 3208970240
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.56090515166105e-06,
+      "loss": 1.251,
+      "theoretical_loss": 3.3321851614750857,
+      "tokens_seen": 3209101312
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.552880757502809e-06,
+      "loss": 1.2014,
+      "theoretical_loss": 3.3321749282972797,
+      "tokens_seen": 3209232384
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.544856363344568e-06,
+      "loss": 1.2863,
+      "theoretical_loss": 3.332164695654429,
+      "tokens_seen": 3209363456
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.536831969186327e-06,
+      "loss": 1.1933,
+      "theoretical_loss": 3.332154463546485,
+      "tokens_seen": 3209494528
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.5288075750280855e-06,
+      "loss": 1.3045,
+      "theoretical_loss": 3.3321442319733965,
+      "tokens_seen": 3209625600
+    },
+    {
+      "epoch": 0.95,
+      "objective/train/docs_used": 1075876,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2660448551177979,
+      "objective/train/theoretical_loss": 3.332134000935114,
+      "objective/train/tokens_used": 1580215776,
+      "theoretical_loss": 3.332134000935114,
+      "tokens_seen": 3209756672
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.5207831808698445e-06,
+      "loss": 1.2257,
+      "theoretical_loss": 3.332134000935114,
+      "tokens_seen": 3209756672
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.5127587867116036e-06,
+      "loss": 1.2672,
+      "theoretical_loss": 3.332123770431589,
+      "tokens_seen": 3209887744
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.504734392553363e-06,
+      "loss": 1.2617,
+      "theoretical_loss": 3.33211354046277,
+      "tokens_seen": 3210018816
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.496709998395122e-06,
+      "loss": 1.2384,
+      "theoretical_loss": 3.3321033110286087,
+      "tokens_seen": 3210149888
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.488685604236881e-06,
+      "loss": 1.205,
+      "theoretical_loss": 3.3320930821290546,
+      "tokens_seen": 3210280960
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.480661210078639e-06,
+      "loss": 1.2144,
+      "theoretical_loss": 3.3320828537640574,
+      "tokens_seen": 3210412032
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.472636815920398e-06,
+      "loss": 1.2001,
+      "theoretical_loss": 3.3320726259335687,
+      "tokens_seen": 3210543104
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.464612421762157e-06,
+      "loss": 1.2499,
+      "theoretical_loss": 3.3320623986375377,
+      "tokens_seen": 3210674176
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.456588027603916e-06,
+      "loss": 1.3016,
+      "theoretical_loss": 3.332052171875915,
+      "tokens_seen": 3210805248
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.448563633445675e-06,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3320419456486503,
+      "tokens_seen": 3210936320
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.440539239287434e-06,
+      "loss": 1.2144,
+      "theoretical_loss": 3.332031719955695,
+      "tokens_seen": 3211067392
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.4325148451291925e-06,
+      "loss": 1.2384,
+      "theoretical_loss": 3.332021494796998,
+      "tokens_seen": 3211198464
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.4244904509709515e-06,
+      "loss": 1.2542,
+      "theoretical_loss": 3.332011270172511,
+      "tokens_seen": 3211329536
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.416466056812711e-06,
+      "loss": 1.2442,
+      "theoretical_loss": 3.3320010460821834,
+      "tokens_seen": 3211460608
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.40844166265447e-06,
+      "loss": 1.1936,
+      "theoretical_loss": 3.331990822525966,
+      "tokens_seen": 3211591680
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.400417268496229e-06,
+      "loss": 1.2228,
+      "theoretical_loss": 3.331980599503808,
+      "tokens_seen": 3211722752
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.392392874337988e-06,
+      "loss": 1.2189,
+      "theoretical_loss": 3.331970377015661,
+      "tokens_seen": 3211853824
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.384368480179747e-06,
+      "loss": 1.1996,
+      "theoretical_loss": 3.331960155061475,
+      "tokens_seen": 3211984896
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.376344086021506e-06,
+      "loss": 1.2072,
+      "theoretical_loss": 3.3319499336411997,
+      "tokens_seen": 3212115968
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.368319691863265e-06,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3319397127547865,
+      "tokens_seen": 3212247040
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.360295297705024e-06,
+      "loss": 1.1397,
+      "theoretical_loss": 3.3319294924021845,
+      "tokens_seen": 3212378112
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.352270903546783e-06,
+      "loss": 1.2079,
+      "theoretical_loss": 3.3319192725833453,
+      "tokens_seen": 3212509184
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.344246509388542e-06,
+      "loss": 1.1849,
+      "theoretical_loss": 3.331909053298218,
+      "tokens_seen": 3212640256
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.3362221152303e-06,
+      "loss": 1.1851,
+      "theoretical_loss": 3.3318988345467537,
+      "tokens_seen": 3212771328
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.328197721072059e-06,
+      "loss": 1.2607,
+      "theoretical_loss": 3.3318886163289028,
+      "tokens_seen": 3212902400
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.3201733269138184e-06,
+      "loss": 1.2736,
+      "theoretical_loss": 3.331878398644615,
+      "tokens_seen": 3213033472
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.312148932755577e-06,
+      "loss": 1.2424,
+      "theoretical_loss": 3.331868181493842,
+      "tokens_seen": 3213164544
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.304124538597336e-06,
+      "loss": 1.3196,
+      "theoretical_loss": 3.3318579648765327,
+      "tokens_seen": 3213295616
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.296100144439095e-06,
+      "loss": 1.237,
+      "theoretical_loss": 3.3318477487926383,
+      "tokens_seen": 3213426688
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.288075750280854e-06,
+      "loss": 1.2552,
+      "theoretical_loss": 3.3318375332421093,
+      "tokens_seen": 3213557760
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.280051356122613e-06,
+      "loss": 1.2841,
+      "theoretical_loss": 3.3318273182248954,
+      "tokens_seen": 3213688832
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.272026961964372e-06,
+      "loss": 1.3036,
+      "theoretical_loss": 3.3318171037409474,
+      "tokens_seen": 3213819904
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.264002567806131e-06,
+      "loss": 1.2279,
+      "theoretical_loss": 3.331806889790216,
+      "tokens_seen": 3213950976
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.25597817364789e-06,
+      "loss": 1.1768,
+      "theoretical_loss": 3.3317966763726514,
+      "tokens_seen": 3214082048
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.247953779489649e-06,
+      "loss": 1.189,
+      "theoretical_loss": 3.331786463488204,
+      "tokens_seen": 3214213120
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.239929385331407e-06,
+      "loss": 1.2124,
+      "theoretical_loss": 3.331776251136824,
+      "tokens_seen": 3214344192
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.231904991173166e-06,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3317660393184623,
+      "tokens_seen": 3214475264
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.2238805970149255e-06,
+      "loss": 1.2209,
+      "theoretical_loss": 3.331755828033069,
+      "tokens_seen": 3214606336
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.2158562028566845e-06,
+      "loss": 1.2795,
+      "theoretical_loss": 3.3317456172805944,
+      "tokens_seen": 3214737408
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.207831808698444e-06,
+      "loss": 1.1963,
+      "theoretical_loss": 3.3317354070609895,
+      "tokens_seen": 3214868480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.199807414540203e-06,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3317251973742046,
+      "tokens_seen": 3214999552
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.191783020381962e-06,
+      "loss": 1.2795,
+      "theoretical_loss": 3.33171498822019,
+      "tokens_seen": 3215130624
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.183758626223721e-06,
+      "loss": 1.189,
+      "theoretical_loss": 3.3317047795988963,
+      "tokens_seen": 3215261696
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.17573423206548e-06,
+      "loss": 1.2439,
+      "theoretical_loss": 3.331694571510274,
+      "tokens_seen": 3215392768
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.167709837907239e-06,
+      "loss": 1.2419,
+      "theoretical_loss": 3.331684363954273,
+      "tokens_seen": 3215523840
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.159685443748997e-06,
+      "loss": 1.2199,
+      "theoretical_loss": 3.331674156930845,
+      "tokens_seen": 3215654912
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.151661049590756e-06,
+      "loss": 1.2197,
+      "theoretical_loss": 3.33166395043994,
+      "tokens_seen": 3215785984
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.143636655432515e-06,
+      "loss": 1.2458,
+      "theoretical_loss": 3.3316537444815078,
+      "tokens_seen": 3215917056
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.135612261274273e-06,
+      "loss": 1.2809,
+      "theoretical_loss": 3.3316435390554995,
+      "tokens_seen": 3216048128
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.1275878671160325e-06,
+      "loss": 1.245,
+      "theoretical_loss": 3.3316333341618654,
+      "tokens_seen": 3216179200
+    },
+    {
+      "epoch": 0.95,
+      "objective/train/docs_used": 1078111,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.3145325183868408,
+      "objective/train/theoretical_loss": 3.3316231298005565,
+      "objective/train/tokens_used": 1586769376,
+      "theoretical_loss": 3.3316231298005565,
+      "tokens_seen": 3216310272
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.1195634729577915e-06,
+      "loss": 1.2477,
+      "theoretical_loss": 3.3316231298005565,
+      "tokens_seen": 3216310272
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.111539078799551e-06,
+      "loss": 1.2829,
+      "theoretical_loss": 3.3316129259715233,
+      "tokens_seen": 3216441344
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.10351468464131e-06,
+      "loss": 1.2079,
+      "theoretical_loss": 3.331602722674716,
+      "tokens_seen": 3216572416
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.095490290483069e-06,
+      "loss": 1.191,
+      "theoretical_loss": 3.3315925199100853,
+      "tokens_seen": 3216703488
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.087465896324828e-06,
+      "loss": 1.2874,
+      "theoretical_loss": 3.3315823176775816,
+      "tokens_seen": 3216834560
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.079441502166587e-06,
+      "loss": 1.2209,
+      "theoretical_loss": 3.331572115977156,
+      "tokens_seen": 3216965632
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.071417108008346e-06,
+      "loss": 1.2817,
+      "theoretical_loss": 3.3315619148087583,
+      "tokens_seen": 3217096704
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.063392713850105e-06,
+      "loss": 1.2037,
+      "theoretical_loss": 3.331551714172339,
+      "tokens_seen": 3217227776
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.055368319691863e-06,
+      "loss": 1.245,
+      "theoretical_loss": 3.33154151406785,
+      "tokens_seen": 3217358848
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.047343925533622e-06,
+      "loss": 1.2579,
+      "theoretical_loss": 3.3315313144952405,
+      "tokens_seen": 3217489920
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.039319531375381e-06,
+      "loss": 1.1893,
+      "theoretical_loss": 3.3315211154544624,
+      "tokens_seen": 3217620992
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.03129513721714e-06,
+      "loss": 1.1832,
+      "theoretical_loss": 3.3315109169454646,
+      "tokens_seen": 3217752064
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.023270743058899e-06,
+      "loss": 1.2861,
+      "theoretical_loss": 3.3315007189681993,
+      "tokens_seen": 3217883136
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.0152463489006585e-06,
+      "loss": 1.2321,
+      "theoretical_loss": 3.3314905215226163,
+      "tokens_seen": 3218014208
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 5.0072219547424175e-06,
+      "loss": 1.2437,
+      "theoretical_loss": 3.331480324608666,
+      "tokens_seen": 3218145280
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.9991975605841766e-06,
+      "loss": 1.2032,
+      "theoretical_loss": 3.3314701282263,
+      "tokens_seen": 3218276352
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.991173166425936e-06,
+      "loss": 1.2061,
+      "theoretical_loss": 3.331459932375468,
+      "tokens_seen": 3218407424
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.983148772267694e-06,
+      "loss": 1.1996,
+      "theoretical_loss": 3.331449737056121,
+      "tokens_seen": 3218538496
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.975124378109453e-06,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3314395422682104,
+      "tokens_seen": 3218669568
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.967099983951212e-06,
+      "loss": 1.1784,
+      "theoretical_loss": 3.331429348011685,
+      "tokens_seen": 3218800640
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.95907558979297e-06,
+      "loss": 1.2277,
+      "theoretical_loss": 3.3314191542864973,
+      "tokens_seen": 3218931712
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.951051195634729e-06,
+      "loss": 1.2095,
+      "theoretical_loss": 3.331408961092597,
+      "tokens_seen": 3219062784
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.943026801476488e-06,
+      "loss": 1.2439,
+      "theoretical_loss": 3.3313987684299353,
+      "tokens_seen": 3219193856
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.935002407318247e-06,
+      "loss": 1.2494,
+      "theoretical_loss": 3.3313885762984627,
+      "tokens_seen": 3219324928
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.926978013160006e-06,
+      "loss": 1.2179,
+      "theoretical_loss": 3.3313783846981297,
+      "tokens_seen": 3219456000
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.9189536190017655e-06,
+      "loss": 1.2356,
+      "theoretical_loss": 3.331368193628887,
+      "tokens_seen": 3219587072
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.9109292248435245e-06,
+      "loss": 1.1781,
+      "theoretical_loss": 3.3313580030906853,
+      "tokens_seen": 3219718144
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.902904830685284e-06,
+      "loss": 1.2332,
+      "theoretical_loss": 3.3313478130834757,
+      "tokens_seen": 3219849216
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.894880436527043e-06,
+      "loss": 1.2564,
+      "theoretical_loss": 3.3313376236072085,
+      "tokens_seen": 3219980288
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.886856042368802e-06,
+      "loss": 1.2051,
+      "theoretical_loss": 3.331327434661835,
+      "tokens_seen": 3220111360
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.878831648210561e-06,
+      "loss": 1.2087,
+      "theoretical_loss": 3.331317246247305,
+      "tokens_seen": 3220242432
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.87080725405232e-06,
+      "loss": 1.2669,
+      "theoretical_loss": 3.3313070583635698,
+      "tokens_seen": 3220373504
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.862782859894078e-06,
+      "loss": 1.2282,
+      "theoretical_loss": 3.33129687101058,
+      "tokens_seen": 3220504576
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.854758465735837e-06,
+      "loss": 1.1886,
+      "theoretical_loss": 3.331286684188287,
+      "tokens_seen": 3220635648
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.846734071577596e-06,
+      "loss": 1.2843,
+      "theoretical_loss": 3.3312764978966407,
+      "tokens_seen": 3220766720
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.838709677419355e-06,
+      "loss": 1.1823,
+      "theoretical_loss": 3.3312663121355923,
+      "tokens_seen": 3220897792
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.830685283261114e-06,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3312561269050924,
+      "tokens_seen": 3221028864
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.822660889102873e-06,
+      "loss": 1.2439,
+      "theoretical_loss": 3.3312459422050917,
+      "tokens_seen": 3221159936
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.8146364949446315e-06,
+      "loss": 1.2322,
+      "theoretical_loss": 3.331235758035541,
+      "tokens_seen": 3221291008
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.806612100786391e-06,
+      "loss": 1.2534,
+      "theoretical_loss": 3.331225574396391,
+      "tokens_seen": 3221422080
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.79858770662815e-06,
+      "loss": 1.1964,
+      "theoretical_loss": 3.3312153912875933,
+      "tokens_seen": 3221553152
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.790563312469909e-06,
+      "loss": 1.2702,
+      "theoretical_loss": 3.3312052087090978,
+      "tokens_seen": 3221684224
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.782538918311668e-06,
+      "loss": 1.2237,
+      "theoretical_loss": 3.331195026660856,
+      "tokens_seen": 3221815296
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.774514524153427e-06,
+      "loss": 1.2126,
+      "theoretical_loss": 3.3311848451428174,
+      "tokens_seen": 3221946368
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.766490129995185e-06,
+      "loss": 1.1746,
+      "theoretical_loss": 3.3311746641549345,
+      "tokens_seen": 3222077440
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.758465735836944e-06,
+      "loss": 1.211,
+      "theoretical_loss": 3.3311644836971572,
+      "tokens_seen": 3222208512
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.750441341678703e-06,
+      "loss": 1.2405,
+      "theoretical_loss": 3.3311543037694364,
+      "tokens_seen": 3222339584
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.742416947520462e-06,
+      "loss": 1.2618,
+      "theoretical_loss": 3.3311441243717232,
+      "tokens_seen": 3222470656
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.734392553362221e-06,
+      "loss": 1.2011,
+      "theoretical_loss": 3.331133945503968,
+      "tokens_seen": 3222601728
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.72636815920398e-06,
+      "loss": 1.2642,
+      "theoretical_loss": 3.3311237671661225,
+      "tokens_seen": 3222732800
+    },
+    {
+      "epoch": 0.95,
+      "objective/train/docs_used": 1080627,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2112478017807007,
+      "objective/train/theoretical_loss": 3.3311135893581367,
+      "objective/train/tokens_used": 1593322976,
+      "theoretical_loss": 3.3311135893581367,
+      "tokens_seen": 3222863872
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.718343765045739e-06,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3311135893581367,
+      "tokens_seen": 3222863872
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.7103193708874985e-06,
+      "loss": 1.211,
+      "theoretical_loss": 3.331103412079962,
+      "tokens_seen": 3222994944
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.7022949767292575e-06,
+      "loss": 1.1599,
+      "theoretical_loss": 3.3310932353315486,
+      "tokens_seen": 3223126016
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.6942705825710166e-06,
+      "loss": 1.2359,
+      "theoretical_loss": 3.331083059112848,
+      "tokens_seen": 3223257088
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.686246188412776e-06,
+      "loss": 1.2648,
+      "theoretical_loss": 3.3310728834238112,
+      "tokens_seen": 3223388160
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.678221794254534e-06,
+      "loss": 1.2054,
+      "theoretical_loss": 3.3310627082643887,
+      "tokens_seen": 3223519232
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.670197400096293e-06,
+      "loss": 1.2396,
+      "theoretical_loss": 3.3310525336345314,
+      "tokens_seen": 3223650304
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.662173005938052e-06,
+      "loss": 1.1999,
+      "theoretical_loss": 3.3310423595341905,
+      "tokens_seen": 3223781376
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.654148611779811e-06,
+      "loss": 1.3446,
+      "theoretical_loss": 3.3310321859633167,
+      "tokens_seen": 3223912448
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.64612421762157e-06,
+      "loss": 1.3702,
+      "theoretical_loss": 3.3310220129218613,
+      "tokens_seen": 3224043520
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.638099823463328e-06,
+      "loss": 1.252,
+      "theoretical_loss": 3.3310118404097744,
+      "tokens_seen": 3224174592
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.630075429305087e-06,
+      "loss": 1.216,
+      "theoretical_loss": 3.3310016684270076,
+      "tokens_seen": 3224305664
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.622051035146846e-06,
+      "loss": 1.2401,
+      "theoretical_loss": 3.330991496973512,
+      "tokens_seen": 3224436736
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.6140266409886055e-06,
+      "loss": 1.2186,
+      "theoretical_loss": 3.330981326049238,
+      "tokens_seen": 3224567808
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.6060022468303645e-06,
+      "loss": 1.2736,
+      "theoretical_loss": 3.3309711556541366,
+      "tokens_seen": 3224698880
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.597977852672124e-06,
+      "loss": 1.191,
+      "theoretical_loss": 3.3309609857881592,
+      "tokens_seen": 3224829952
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.589953458513883e-06,
+      "loss": 1.1659,
+      "theoretical_loss": 3.3309508164512565,
+      "tokens_seen": 3224961024
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.581929064355641e-06,
+      "loss": 1.245,
+      "theoretical_loss": 3.33094064764338,
+      "tokens_seen": 3225092096
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.5739046701974e-06,
+      "loss": 1.2767,
+      "theoretical_loss": 3.330930479364479,
+      "tokens_seen": 3225223168
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.565880276039159e-06,
+      "loss": 1.3042,
+      "theoretical_loss": 3.3309203116145065,
+      "tokens_seen": 3225354240
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.557855881880918e-06,
+      "loss": 1.1928,
+      "theoretical_loss": 3.3309101443934126,
+      "tokens_seen": 3225485312
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.549831487722677e-06,
+      "loss": 1.2858,
+      "theoretical_loss": 3.330899977701148,
+      "tokens_seen": 3225616384
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.541807093564436e-06,
+      "loss": 1.2628,
+      "theoretical_loss": 3.3308898115376646,
+      "tokens_seen": 3225747456
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.533782699406195e-06,
+      "loss": 1.2146,
+      "theoretical_loss": 3.3308796459029124,
+      "tokens_seen": 3225878528
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.525758305247954e-06,
+      "loss": 1.2658,
+      "theoretical_loss": 3.330869480796843,
+      "tokens_seen": 3226009600
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.517733911089713e-06,
+      "loss": 1.228,
+      "theoretical_loss": 3.3308593162194073,
+      "tokens_seen": 3226140672
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.509709516931472e-06,
+      "loss": 1.234,
+      "theoretical_loss": 3.3308491521705563,
+      "tokens_seen": 3226271744
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.5016851227732314e-06,
+      "loss": 1.2633,
+      "theoretical_loss": 3.330838988650241,
+      "tokens_seen": 3226402816
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.4936607286149905e-06,
+      "loss": 1.2369,
+      "theoretical_loss": 3.330828825658413,
+      "tokens_seen": 3226533888
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.485636334456749e-06,
+      "loss": 1.1313,
+      "theoretical_loss": 3.3308186631950227,
+      "tokens_seen": 3226664960
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.477611940298508e-06,
+      "loss": 1.2359,
+      "theoretical_loss": 3.3308085012600213,
+      "tokens_seen": 3226796032
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.469587546140267e-06,
+      "loss": 1.2324,
+      "theoretical_loss": 3.33079833985336,
+      "tokens_seen": 3226927104
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.461563151982025e-06,
+      "loss": 1.254,
+      "theoretical_loss": 3.3307881789749896,
+      "tokens_seen": 3227058176
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.453538757823784e-06,
+      "loss": 1.2479,
+      "theoretical_loss": 3.3307780186248612,
+      "tokens_seen": 3227189248
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.445514363665543e-06,
+      "loss": 1.2707,
+      "theoretical_loss": 3.3307678588029264,
+      "tokens_seen": 3227320320
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.437489969507302e-06,
+      "loss": 1.2169,
+      "theoretical_loss": 3.330757699509136,
+      "tokens_seen": 3227451392
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.429465575349061e-06,
+      "loss": 1.2032,
+      "theoretical_loss": 3.330747540743441,
+      "tokens_seen": 3227582464
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.42144118119082e-06,
+      "loss": 1.2784,
+      "theoretical_loss": 3.3307373825057924,
+      "tokens_seen": 3227713536
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.413416787032579e-06,
+      "loss": 1.2021,
+      "theoretical_loss": 3.3307272247961413,
+      "tokens_seen": 3227844608
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.4053923928743385e-06,
+      "loss": 1.2599,
+      "theoretical_loss": 3.3307170676144393,
+      "tokens_seen": 3227975680
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.3973679987160975e-06,
+      "loss": 1.1588,
+      "theoretical_loss": 3.330706910960637,
+      "tokens_seen": 3228106752
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.389343604557856e-06,
+      "loss": 1.2882,
+      "theoretical_loss": 3.330696754834686,
+      "tokens_seen": 3228237824
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.381319210399615e-06,
+      "loss": 1.3036,
+      "theoretical_loss": 3.330686599236537,
+      "tokens_seen": 3228368896
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.373294816241374e-06,
+      "loss": 1.2039,
+      "theoretical_loss": 3.330676444166141,
+      "tokens_seen": 3228499968
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.365270422083133e-06,
+      "loss": 1.2383,
+      "theoretical_loss": 3.3306662896234496,
+      "tokens_seen": 3228631040
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.357246027924892e-06,
+      "loss": 1.2591,
+      "theoretical_loss": 3.330656135608414,
+      "tokens_seen": 3228762112
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.349221633766651e-06,
+      "loss": 1.2917,
+      "theoretical_loss": 3.330645982120985,
+      "tokens_seen": 3228893184
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.34119723960841e-06,
+      "loss": 1.2157,
+      "theoretical_loss": 3.3306358291611144,
+      "tokens_seen": 3229024256
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.333172845450169e-06,
+      "loss": 1.3017,
+      "theoretical_loss": 3.3306256767287525,
+      "tokens_seen": 3229155328
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.325148451291928e-06,
+      "loss": 1.3736,
+      "theoretical_loss": 3.330615524823851,
+      "tokens_seen": 3229286400
+    },
+    {
+      "epoch": 0.96,
+      "objective/train/docs_used": 1082871,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2482061386108398,
+      "objective/train/theoretical_loss": 3.330605373446361,
+      "objective/train/tokens_used": 1599876576,
+      "theoretical_loss": 3.330605373446361,
+      "tokens_seen": 3229417472
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.317124057133686e-06,
+      "loss": 1.2641,
+      "theoretical_loss": 3.330605373446361,
+      "tokens_seen": 3229417472
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.3090996629754455e-06,
+      "loss": 1.2313,
+      "theoretical_loss": 3.3305952225962336,
+      "tokens_seen": 3229548544
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.3010752688172045e-06,
+      "loss": 1.2199,
+      "theoretical_loss": 3.3305850722734203,
+      "tokens_seen": 3229679616
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.293050874658963e-06,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3305749224778722,
+      "tokens_seen": 3229810688
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.285026480500722e-06,
+      "loss": 1.3572,
+      "theoretical_loss": 3.33056477320954,
+      "tokens_seen": 3229941760
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.277002086342481e-06,
+      "loss": 1.1789,
+      "theoretical_loss": 3.3305546244683755,
+      "tokens_seen": 3230072832
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.26897769218424e-06,
+      "loss": 1.1696,
+      "theoretical_loss": 3.33054447625433,
+      "tokens_seen": 3230203904
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.260953298025999e-06,
+      "loss": 1.2467,
+      "theoretical_loss": 3.330534328567354,
+      "tokens_seen": 3230334976
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.252928903867758e-06,
+      "loss": 1.2373,
+      "theoretical_loss": 3.3305241814074,
+      "tokens_seen": 3230466048
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.244904509709517e-06,
+      "loss": 1.252,
+      "theoretical_loss": 3.3305140347744175,
+      "tokens_seen": 3230597120
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.236880115551276e-06,
+      "loss": 1.2737,
+      "theoretical_loss": 3.3305038886683596,
+      "tokens_seen": 3230728192
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.228855721393035e-06,
+      "loss": 1.2447,
+      "theoretical_loss": 3.3304937430891766,
+      "tokens_seen": 3230859264
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.220831327234794e-06,
+      "loss": 1.3136,
+      "theoretical_loss": 3.3304835980368193,
+      "tokens_seen": 3230990336
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.212806933076553e-06,
+      "loss": 1.2268,
+      "theoretical_loss": 3.33047345351124,
+      "tokens_seen": 3231121408
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.2047825389183115e-06,
+      "loss": 1.1618,
+      "theoretical_loss": 3.3304633095123894,
+      "tokens_seen": 3231252480
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.196758144760071e-06,
+      "loss": 1.2401,
+      "theoretical_loss": 3.330453166040219,
+      "tokens_seen": 3231383552
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.18873375060183e-06,
+      "loss": 1.2274,
+      "theoretical_loss": 3.3304430230946798,
+      "tokens_seen": 3231514624
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.180709356443589e-06,
+      "loss": 1.1988,
+      "theoretical_loss": 3.3304328806757235,
+      "tokens_seen": 3231645696
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.172684962285348e-06,
+      "loss": 1.2604,
+      "theoretical_loss": 3.330422738783301,
+      "tokens_seen": 3231776768
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.164660568127107e-06,
+      "loss": 1.2108,
+      "theoretical_loss": 3.3304125974173644,
+      "tokens_seen": 3231907840
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.156636173968866e-06,
+      "loss": 1.2226,
+      "theoretical_loss": 3.330402456577864,
+      "tokens_seen": 3232038912
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.148611779810625e-06,
+      "loss": 1.2772,
+      "theoretical_loss": 3.3303923162647515,
+      "tokens_seen": 3232169984
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.140587385652383e-06,
+      "loss": 1.2131,
+      "theoretical_loss": 3.330382176477978,
+      "tokens_seen": 3232301056
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.132562991494142e-06,
+      "loss": 1.2349,
+      "theoretical_loss": 3.330372037217496,
+      "tokens_seen": 3232432128
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.124538597335901e-06,
+      "loss": 1.3149,
+      "theoretical_loss": 3.3303618984832553,
+      "tokens_seen": 3232563200
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.11651420317766e-06,
+      "loss": 1.2555,
+      "theoretical_loss": 3.3303517602752084,
+      "tokens_seen": 3232694272
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.1084898090194186e-06,
+      "loss": 1.1729,
+      "theoretical_loss": 3.3303416225933056,
+      "tokens_seen": 3232825344
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.100465414861178e-06,
+      "loss": 1.1876,
+      "theoretical_loss": 3.3303314854374992,
+      "tokens_seen": 3232956416
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.092441020702937e-06,
+      "loss": 1.2608,
+      "theoretical_loss": 3.33032134880774,
+      "tokens_seen": 3233087488
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.084416626544696e-06,
+      "loss": 1.1884,
+      "theoretical_loss": 3.3303112127039802,
+      "tokens_seen": 3233218560
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.076392232386455e-06,
+      "loss": 1.2289,
+      "theoretical_loss": 3.33030107712617,
+      "tokens_seen": 3233349632
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.068367838228214e-06,
+      "loss": 1.2794,
+      "theoretical_loss": 3.330290942074262,
+      "tokens_seen": 3233480704
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.060343444069973e-06,
+      "loss": 1.2816,
+      "theoretical_loss": 3.330280807548206,
+      "tokens_seen": 3233611776
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.052319049911732e-06,
+      "loss": 1.315,
+      "theoretical_loss": 3.330270673547955,
+      "tokens_seen": 3233742848
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.044294655753491e-06,
+      "loss": 1.2776,
+      "theoretical_loss": 3.3302605400734597,
+      "tokens_seen": 3233873920
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.03627026159525e-06,
+      "loss": 1.1486,
+      "theoretical_loss": 3.3302504071246717,
+      "tokens_seen": 3234004992
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.028245867437009e-06,
+      "loss": 1.2304,
+      "theoretical_loss": 3.3302402747015423,
+      "tokens_seen": 3234136064
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.020221473278768e-06,
+      "loss": 1.2525,
+      "theoretical_loss": 3.330230142804023,
+      "tokens_seen": 3234267136
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.012197079120526e-06,
+      "loss": 1.2662,
+      "theoretical_loss": 3.330220011432065,
+      "tokens_seen": 3234398208
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 4.0041726849622855e-06,
+      "loss": 1.2042,
+      "theoretical_loss": 3.33020988058562,
+      "tokens_seen": 3234529280
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.9961482908040445e-06,
+      "loss": 1.2385,
+      "theoretical_loss": 3.330199750264639,
+      "tokens_seen": 3234660352
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.988123896645804e-06,
+      "loss": 1.2346,
+      "theoretical_loss": 3.3301896204690746,
+      "tokens_seen": 3234791424
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.980099502487563e-06,
+      "loss": 1.2298,
+      "theoretical_loss": 3.330179491198877,
+      "tokens_seen": 3234922496
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.972075108329322e-06,
+      "loss": 1.2381,
+      "theoretical_loss": 3.330169362453998,
+      "tokens_seen": 3235053568
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.96405071417108e-06,
+      "loss": 1.2759,
+      "theoretical_loss": 3.3301592342343898,
+      "tokens_seen": 3235184640
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.956026320012839e-06,
+      "loss": 1.2387,
+      "theoretical_loss": 3.330149106540003,
+      "tokens_seen": 3235315712
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.948001925854598e-06,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3301389793707896,
+      "tokens_seen": 3235446784
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.939977531696357e-06,
+      "loss": 1.154,
+      "theoretical_loss": 3.3301288527267006,
+      "tokens_seen": 3235577856
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.931953137538116e-06,
+      "loss": 1.188,
+      "theoretical_loss": 3.3301187266076875,
+      "tokens_seen": 3235708928
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.923928743379875e-06,
+      "loss": 1.2877,
+      "theoretical_loss": 3.330108601013703,
+      "tokens_seen": 3235840000
+    },
+    {
+      "epoch": 0.96,
+      "objective/train/docs_used": 1085025,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.261237382888794,
+      "objective/train/theoretical_loss": 3.330098475944697,
+      "objective/train/tokens_used": 1606430176,
+      "theoretical_loss": 3.330098475944697,
+      "tokens_seen": 3235971072
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.9159043492216334e-06,
+      "loss": 1.2423,
+      "theoretical_loss": 3.330098475944697,
+      "tokens_seen": 3235971072
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.9078799550633925e-06,
+      "loss": 1.2539,
+      "theoretical_loss": 3.330088351400622,
+      "tokens_seen": 3236102144
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.8998555609051516e-06,
+      "loss": 1.274,
+      "theoretical_loss": 3.330078227381429,
+      "tokens_seen": 3236233216
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.891831166746911e-06,
+      "loss": 1.2197,
+      "theoretical_loss": 3.33006810388707,
+      "tokens_seen": 3236364288
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.88380677258867e-06,
+      "loss": 1.1949,
+      "theoretical_loss": 3.3300579809174966,
+      "tokens_seen": 3236495360
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.875782378430429e-06,
+      "loss": 1.147,
+      "theoretical_loss": 3.3300478584726596,
+      "tokens_seen": 3236626432
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.867757984272188e-06,
+      "loss": 1.2278,
+      "theoretical_loss": 3.3300377365525113,
+      "tokens_seen": 3236757504
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.859733590113947e-06,
+      "loss": 1.1977,
+      "theoretical_loss": 3.330027615157003,
+      "tokens_seen": 3236888576
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.851709195955706e-06,
+      "loss": 1.2922,
+      "theoretical_loss": 3.3300174942860856,
+      "tokens_seen": 3237019648
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.843684801797465e-06,
+      "loss": 1.1435,
+      "theoretical_loss": 3.3300073739397122,
+      "tokens_seen": 3237150720
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.835660407639224e-06,
+      "loss": 1.2622,
+      "theoretical_loss": 3.329997254117833,
+      "tokens_seen": 3237281792
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.827636013480982e-06,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3299871348204,
+      "tokens_seen": 3237412864
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.819611619322741e-06,
+      "loss": 1.2397,
+      "theoretical_loss": 3.3299770160473656,
+      "tokens_seen": 3237543936
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.8115872251645e-06,
+      "loss": 1.2868,
+      "theoretical_loss": 3.32996689779868,
+      "tokens_seen": 3237675008
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.803562831006259e-06,
+      "loss": 1.1662,
+      "theoretical_loss": 3.3299567800742955,
+      "tokens_seen": 3237806080
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.795538436848018e-06,
+      "loss": 1.2175,
+      "theoretical_loss": 3.329946662874164,
+      "tokens_seen": 3237937152
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.787514042689777e-06,
+      "loss": 1.1656,
+      "theoretical_loss": 3.3299365461982364,
+      "tokens_seen": 3238068224
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.779489648531536e-06,
+      "loss": 1.2048,
+      "theoretical_loss": 3.329926430046465,
+      "tokens_seen": 3238199296
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.7714652543732952e-06,
+      "loss": 1.2034,
+      "theoretical_loss": 3.329916314418801,
+      "tokens_seen": 3238330368
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.763440860215054e-06,
+      "loss": 1.2484,
+      "theoretical_loss": 3.329906199315196,
+      "tokens_seen": 3238461440
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.755416466056813e-06,
+      "loss": 1.2534,
+      "theoretical_loss": 3.3298960847356023,
+      "tokens_seen": 3238592512
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.747392071898572e-06,
+      "loss": 1.1937,
+      "theoretical_loss": 3.3298859706799706,
+      "tokens_seen": 3238723584
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.739367677740331e-06,
+      "loss": 1.1728,
+      "theoretical_loss": 3.329875857148253,
+      "tokens_seen": 3238854656
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.7313432835820893e-06,
+      "loss": 1.2028,
+      "theoretical_loss": 3.3298657441404016,
+      "tokens_seen": 3238985728
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.7233188894238483e-06,
+      "loss": 1.1989,
+      "theoretical_loss": 3.3298556316563674,
+      "tokens_seen": 3239116800
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.7152944952656074e-06,
+      "loss": 1.1997,
+      "theoretical_loss": 3.3298455196961028,
+      "tokens_seen": 3239247872
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.7072701011073664e-06,
+      "loss": 1.23,
+      "theoretical_loss": 3.3298354082595583,
+      "tokens_seen": 3239378944
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6992457069491255e-06,
+      "loss": 1.2113,
+      "theoretical_loss": 3.3298252973466864,
+      "tokens_seen": 3239510016
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6912213127908845e-06,
+      "loss": 1.1842,
+      "theoretical_loss": 3.329815186957439,
+      "tokens_seen": 3239641088
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6831969186326436e-06,
+      "loss": 1.262,
+      "theoretical_loss": 3.329805077091767,
+      "tokens_seen": 3239772160
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6751725244744022e-06,
+      "loss": 1.2522,
+      "theoretical_loss": 3.3297949677496232,
+      "tokens_seen": 3239903232
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6671481303161613e-06,
+      "loss": 1.1707,
+      "theoretical_loss": 3.3297848589309584,
+      "tokens_seen": 3240034304
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6591237361579204e-06,
+      "loss": 1.1635,
+      "theoretical_loss": 3.3297747506357247,
+      "tokens_seen": 3240165376
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6510993419996794e-06,
+      "loss": 1.2576,
+      "theoretical_loss": 3.329764642863873,
+      "tokens_seen": 3240296448
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6430749478414385e-06,
+      "loss": 1.1994,
+      "theoretical_loss": 3.329754535615357,
+      "tokens_seen": 3240427520
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6350505536831967e-06,
+      "loss": 1.1859,
+      "theoretical_loss": 3.3297444288901263,
+      "tokens_seen": 3240558592
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.6270261595249557e-06,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3297343226881337,
+      "tokens_seen": 3240689664
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.619001765366715e-06,
+      "loss": 1.2214,
+      "theoretical_loss": 3.329724217009331,
+      "tokens_seen": 3240820736
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.610977371208474e-06,
+      "loss": 1.1364,
+      "theoretical_loss": 3.32971411185367,
+      "tokens_seen": 3240951808
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.602952977050233e-06,
+      "loss": 1.2464,
+      "theoretical_loss": 3.3297040072211015,
+      "tokens_seen": 3241082880
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.594928582891992e-06,
+      "loss": 1.2391,
+      "theoretical_loss": 3.3296939031115786,
+      "tokens_seen": 3241213952
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.5869041887337506e-06,
+      "loss": 1.3281,
+      "theoretical_loss": 3.329683799525052,
+      "tokens_seen": 3241345024
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.5788797945755097e-06,
+      "loss": 1.2344,
+      "theoretical_loss": 3.3296736964614744,
+      "tokens_seen": 3241476096
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.5708554004172687e-06,
+      "loss": 1.2879,
+      "theoretical_loss": 3.329663593920797,
+      "tokens_seen": 3241607168
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.562831006259028e-06,
+      "loss": 1.2297,
+      "theoretical_loss": 3.3296534919029717,
+      "tokens_seen": 3241738240
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.554806612100787e-06,
+      "loss": 1.1589,
+      "theoretical_loss": 3.3296433904079503,
+      "tokens_seen": 3241869312
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.546782217942546e-06,
+      "loss": 1.1956,
+      "theoretical_loss": 3.3296332894356846,
+      "tokens_seen": 3242000384
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.538757823784304e-06,
+      "loss": 1.2153,
+      "theoretical_loss": 3.3296231889861265,
+      "tokens_seen": 3242131456
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.530733429626063e-06,
+      "loss": 1.1717,
+      "theoretical_loss": 3.3296130890592277,
+      "tokens_seen": 3242262528
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.5227090354678222e-06,
+      "loss": 1.1798,
+      "theoretical_loss": 3.32960298965494,
+      "tokens_seen": 3242393600
+    },
+    {
+      "epoch": 0.97,
+      "objective/train/docs_used": 1087175,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2268452644348145,
+      "objective/train/theoretical_loss": 3.329592890773216,
+      "objective/train/tokens_used": 1612983776,
+      "theoretical_loss": 3.329592890773216,
+      "tokens_seen": 3242524672
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.5146846413095813e-06,
+      "loss": 1.2063,
+      "theoretical_loss": 3.329592890773216,
+      "tokens_seen": 3242524672
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.5066602471513404e-06,
+      "loss": 1.1869,
+      "theoretical_loss": 3.3295827924140062,
+      "tokens_seen": 3242655744
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.498635852993099e-06,
+      "loss": 1.1797,
+      "theoretical_loss": 3.329572694577263,
+      "tokens_seen": 3242786816
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.490611458834858e-06,
+      "loss": 1.2496,
+      "theoretical_loss": 3.329562597262939,
+      "tokens_seen": 3242917888
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.482587064676617e-06,
+      "loss": 1.2864,
+      "theoretical_loss": 3.329552500470985,
+      "tokens_seen": 3243048960
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.474562670518376e-06,
+      "loss": 1.227,
+      "theoretical_loss": 3.3295424042013533,
+      "tokens_seen": 3243180032
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4665382763601352e-06,
+      "loss": 1.1978,
+      "theoretical_loss": 3.329532308453996,
+      "tokens_seen": 3243311104
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4585138822018943e-06,
+      "loss": 1.2707,
+      "theoretical_loss": 3.3295222132288647,
+      "tokens_seen": 3243442176
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4504894880436533e-06,
+      "loss": 1.2519,
+      "theoretical_loss": 3.329512118525911,
+      "tokens_seen": 3243573248
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4424650938854116e-06,
+      "loss": 1.253,
+      "theoretical_loss": 3.3295020243450875,
+      "tokens_seen": 3243704320
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4344406997271706e-06,
+      "loss": 1.1864,
+      "theoretical_loss": 3.3294919306863457,
+      "tokens_seen": 3243835392
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4264163055689297e-06,
+      "loss": 1.1697,
+      "theoretical_loss": 3.329481837549637,
+      "tokens_seen": 3243966464
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4183919114106887e-06,
+      "loss": 1.258,
+      "theoretical_loss": 3.3294717449349145,
+      "tokens_seen": 3244097536
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4103675172524474e-06,
+      "loss": 1.2076,
+      "theoretical_loss": 3.329461652842129,
+      "tokens_seen": 3244228608
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.4023431230942064e-06,
+      "loss": 1.282,
+      "theoretical_loss": 3.329451561271233,
+      "tokens_seen": 3244359680
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.3943187289359655e-06,
+      "loss": 1.2488,
+      "theoretical_loss": 3.3294414702221786,
+      "tokens_seen": 3244490752
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.3862943347777246e-06,
+      "loss": 1.2902,
+      "theoretical_loss": 3.3294313796949173,
+      "tokens_seen": 3244621824
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.3782699406194836e-06,
+      "loss": 1.2922,
+      "theoretical_loss": 3.329421289689401,
+      "tokens_seen": 3244752896
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.3702455464612427e-06,
+      "loss": 1.1355,
+      "theoretical_loss": 3.329411200205582,
+      "tokens_seen": 3244883968
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.3622211523030017e-06,
+      "loss": 1.2639,
+      "theoretical_loss": 3.3294011112434116,
+      "tokens_seen": 3245015040
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.35419675814476e-06,
+      "loss": 1.2339,
+      "theoretical_loss": 3.3293910228028425,
+      "tokens_seen": 3245146112
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.346172363986519e-06,
+      "loss": 1.2154,
+      "theoretical_loss": 3.329380934883827,
+      "tokens_seen": 3245277184
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.338147969828278e-06,
+      "loss": 1.2098,
+      "theoretical_loss": 3.3293708474863157,
+      "tokens_seen": 3245408256
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.330123575670037e-06,
+      "loss": 1.2434,
+      "theoretical_loss": 3.3293607606102618,
+      "tokens_seen": 3245539328
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.3220991815117958e-06,
+      "loss": 1.2479,
+      "theoretical_loss": 3.3293506742556165,
+      "tokens_seen": 3245670400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.314074787353555e-06,
+      "loss": 1.2868,
+      "theoretical_loss": 3.3293405884223324,
+      "tokens_seen": 3245801472
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.306050393195314e-06,
+      "loss": 1.1997,
+      "theoretical_loss": 3.329330503110361,
+      "tokens_seen": 3245932544
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.298025999037073e-06,
+      "loss": 1.1901,
+      "theoretical_loss": 3.329320418319655,
+      "tokens_seen": 3246063616
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.290001604878832e-06,
+      "loss": 1.2416,
+      "theoretical_loss": 3.3293103340501657,
+      "tokens_seen": 3246194688
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.281977210720591e-06,
+      "loss": 1.2418,
+      "theoretical_loss": 3.3293002503018454,
+      "tokens_seen": 3246325760
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.27395281656235e-06,
+      "loss": 1.2428,
+      "theoretical_loss": 3.329290167074646,
+      "tokens_seen": 3246456832
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.265928422404109e-06,
+      "loss": 1.2583,
+      "theoretical_loss": 3.3292800843685195,
+      "tokens_seen": 3246587904
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.2579040282458674e-06,
+      "loss": 1.246,
+      "theoretical_loss": 3.3292700021834185,
+      "tokens_seen": 3246718976
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.2498796340876264e-06,
+      "loss": 1.18,
+      "theoretical_loss": 3.329259920519294,
+      "tokens_seen": 3246850048
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.241855239929385e-06,
+      "loss": 1.2441,
+      "theoretical_loss": 3.329249839376099,
+      "tokens_seen": 3246981120
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.233830845771144e-06,
+      "loss": 1.18,
+      "theoretical_loss": 3.3292397587537854,
+      "tokens_seen": 3247112192
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.225806451612903e-06,
+      "loss": 1.2196,
+      "theoretical_loss": 3.329229678652305,
+      "tokens_seen": 3247243264
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.2177820574546622e-06,
+      "loss": 1.2221,
+      "theoretical_loss": 3.32921959907161,
+      "tokens_seen": 3247374336
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.2097576632964213e-06,
+      "loss": 1.2213,
+      "theoretical_loss": 3.329209520011652,
+      "tokens_seen": 3247505408
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.2017332691381804e-06,
+      "loss": 1.244,
+      "theoretical_loss": 3.329199441472384,
+      "tokens_seen": 3247636480
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1937088749799394e-06,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3291893634537573,
+      "tokens_seen": 3247767552
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1856844808216985e-06,
+      "loss": 1.2022,
+      "theoretical_loss": 3.3291792859557248,
+      "tokens_seen": 3247898624
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.177660086663457e-06,
+      "loss": 1.1842,
+      "theoretical_loss": 3.329169208978237,
+      "tokens_seen": 3248029696
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.169635692505216e-06,
+      "loss": 1.1942,
+      "theoretical_loss": 3.329159132521248,
+      "tokens_seen": 3248160768
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.161611298346975e-06,
+      "loss": 1.1766,
+      "theoretical_loss": 3.3291490565847086,
+      "tokens_seen": 3248291840
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1535869041887334e-06,
+      "loss": 1.2018,
+      "theoretical_loss": 3.3291389811685717,
+      "tokens_seen": 3248422912
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1455625100304925e-06,
+      "loss": 1.2385,
+      "theoretical_loss": 3.329128906272789,
+      "tokens_seen": 3248553984
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1375381158722516e-06,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3291188318973126,
+      "tokens_seen": 3248685056
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1295137217140106e-06,
+      "loss": 1.1575,
+      "theoretical_loss": 3.3291087580420946,
+      "tokens_seen": 3248816128
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1214893275557697e-06,
+      "loss": 1.2002,
+      "theoretical_loss": 3.329098684707087,
+      "tokens_seen": 3248947200
+    },
+    {
+      "epoch": 0.97,
+      "objective/train/docs_used": 1089234,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2460660934448242,
+      "objective/train/theoretical_loss": 3.3290886118922427,
+      "objective/train/tokens_used": 1619537376,
+      "theoretical_loss": 3.3290886118922427,
+      "tokens_seen": 3249078272
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.1134649333975287e-06,
+      "loss": 1.2457,
+      "theoretical_loss": 3.3290886118922427,
+      "tokens_seen": 3249078272
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.105440539239288e-06,
+      "loss": 1.1854,
+      "theoretical_loss": 3.3290785395975133,
+      "tokens_seen": 3249209344
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.097416145081047e-06,
+      "loss": 1.1573,
+      "theoretical_loss": 3.329068467822851,
+      "tokens_seen": 3249340416
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.0893917509228055e-06,
+      "loss": 1.2177,
+      "theoretical_loss": 3.329058396568208,
+      "tokens_seen": 3249471488
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.0813673567645646e-06,
+      "loss": 1.2669,
+      "theoretical_loss": 3.3290483258335364,
+      "tokens_seen": 3249602560
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.073342962606323e-06,
+      "loss": 1.2521,
+      "theoretical_loss": 3.3290382556187885,
+      "tokens_seen": 3249733632
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.0653185684480823e-06,
+      "loss": 1.2003,
+      "theoretical_loss": 3.3290281859239164,
+      "tokens_seen": 3249864704
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.0572941742898413e-06,
+      "loss": 1.1793,
+      "theoretical_loss": 3.3290181167488724,
+      "tokens_seen": 3249995776
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.0492697801316004e-06,
+      "loss": 1.1881,
+      "theoretical_loss": 3.329008048093609,
+      "tokens_seen": 3250126848
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.041245385973359e-06,
+      "loss": 1.2236,
+      "theoretical_loss": 3.3289979799580776,
+      "tokens_seen": 3250257920
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.033220991815118e-06,
+      "loss": 1.2476,
+      "theoretical_loss": 3.328987912342231,
+      "tokens_seen": 3250388992
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.025196597656877e-06,
+      "loss": 1.2211,
+      "theoretical_loss": 3.3289778452460217,
+      "tokens_seen": 3250520064
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.017172203498636e-06,
+      "loss": 1.2443,
+      "theoretical_loss": 3.328967778669401,
+      "tokens_seen": 3250651136
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.0091478093403952e-06,
+      "loss": 1.2252,
+      "theoretical_loss": 3.328957712612322,
+      "tokens_seen": 3250782208
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.001123415182154e-06,
+      "loss": 1.3461,
+      "theoretical_loss": 3.3289476470747363,
+      "tokens_seen": 3250913280
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9930990210239125e-06,
+      "loss": 1.2019,
+      "theoretical_loss": 3.3289375820565965,
+      "tokens_seen": 3251044352
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9850746268656716e-06,
+      "loss": 1.3184,
+      "theoretical_loss": 3.328927517557855,
+      "tokens_seen": 3251175424
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9770502327074306e-06,
+      "loss": 1.2627,
+      "theoretical_loss": 3.328917453578464,
+      "tokens_seen": 3251306496
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9690258385491897e-06,
+      "loss": 1.2125,
+      "theoretical_loss": 3.328907390118375,
+      "tokens_seen": 3251437568
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9610014443909487e-06,
+      "loss": 1.2683,
+      "theoretical_loss": 3.3288973271775415,
+      "tokens_seen": 3251568640
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9529770502327074e-06,
+      "loss": 1.1001,
+      "theoretical_loss": 3.328887264755915,
+      "tokens_seen": 3251699712
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9449526560744664e-06,
+      "loss": 1.2178,
+      "theoretical_loss": 3.3288772028534477,
+      "tokens_seen": 3251830784
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9369282619162255e-06,
+      "loss": 1.1798,
+      "theoretical_loss": 3.3288671414700923,
+      "tokens_seen": 3251961856
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9289038677579846e-06,
+      "loss": 1.2599,
+      "theoretical_loss": 3.3288570806058013,
+      "tokens_seen": 3252092928
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9208794735997436e-06,
+      "loss": 1.2363,
+      "theoretical_loss": 3.3288470202605263,
+      "tokens_seen": 3252224000
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.9128550794415023e-06,
+      "loss": 1.2746,
+      "theoretical_loss": 3.32883696043422,
+      "tokens_seen": 3252355072
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.904830685283261e-06,
+      "loss": 1.1878,
+      "theoretical_loss": 3.3288269011268348,
+      "tokens_seen": 3252486144
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.89680629112502e-06,
+      "loss": 1.1783,
+      "theoretical_loss": 3.3288168423383224,
+      "tokens_seen": 3252617216
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.888781896966779e-06,
+      "loss": 1.1709,
+      "theoretical_loss": 3.328806784068636,
+      "tokens_seen": 3252748288
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.880757502808538e-06,
+      "loss": 1.2469,
+      "theoretical_loss": 3.328796726317728,
+      "tokens_seen": 3252879360
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.872733108650297e-06,
+      "loss": 1.1718,
+      "theoretical_loss": 3.3287866690855497,
+      "tokens_seen": 3253010432
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.864708714492056e-06,
+      "loss": 1.2498,
+      "theoretical_loss": 3.328776612372054,
+      "tokens_seen": 3253141504
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.856684320333815e-06,
+      "loss": 1.166,
+      "theoretical_loss": 3.3287665561771935,
+      "tokens_seen": 3253272576
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.848659926175574e-06,
+      "loss": 1.311,
+      "theoretical_loss": 3.3287565005009205,
+      "tokens_seen": 3253403648
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.840635532017333e-06,
+      "loss": 1.2068,
+      "theoretical_loss": 3.328746445343187,
+      "tokens_seen": 3253534720
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.832611137859092e-06,
+      "loss": 1.1949,
+      "theoretical_loss": 3.328736390703946,
+      "tokens_seen": 3253665792
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.8245867437008506e-06,
+      "loss": 1.2617,
+      "theoretical_loss": 3.328726336583149,
+      "tokens_seen": 3253796864
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.8165623495426097e-06,
+      "loss": 1.2206,
+      "theoretical_loss": 3.328716282980749,
+      "tokens_seen": 3253927936
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.8085379553843683e-06,
+      "loss": 1.2921,
+      "theoretical_loss": 3.328706229896698,
+      "tokens_seen": 3254059008
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.8005135612261274e-06,
+      "loss": 1.1811,
+      "theoretical_loss": 3.328696177330949,
+      "tokens_seen": 3254190080
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7924891670678864e-06,
+      "loss": 1.2431,
+      "theoretical_loss": 3.328686125283454,
+      "tokens_seen": 3254321152
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7844647729096455e-06,
+      "loss": 1.2938,
+      "theoretical_loss": 3.3286760737541656,
+      "tokens_seen": 3254452224
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7764403787514046e-06,
+      "loss": 1.248,
+      "theoretical_loss": 3.328666022743036,
+      "tokens_seen": 3254583296
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7684159845931636e-06,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3286559722500177,
+      "tokens_seen": 3254714368
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7603915904349223e-06,
+      "loss": 1.2063,
+      "theoretical_loss": 3.328645922275063,
+      "tokens_seen": 3254845440
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7523671962766813e-06,
+      "loss": 1.1575,
+      "theoretical_loss": 3.3286358728181247,
+      "tokens_seen": 3254976512
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7443428021184404e-06,
+      "loss": 1.2303,
+      "theoretical_loss": 3.3286258238791544,
+      "tokens_seen": 3255107584
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.736318407960199e-06,
+      "loss": 1.1952,
+      "theoretical_loss": 3.328615775458106,
+      "tokens_seen": 3255238656
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.728294013801958e-06,
+      "loss": 1.2624,
+      "theoretical_loss": 3.3286057275549306,
+      "tokens_seen": 3255369728
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.720269619643717e-06,
+      "loss": 1.1865,
+      "theoretical_loss": 3.328595680169581,
+      "tokens_seen": 3255500800
+    },
+    {
+      "epoch": 0.97,
+      "objective/train/docs_used": 1091411,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2188622951507568,
+      "objective/train/theoretical_loss": 3.3285856333020103,
+      "objective/train/tokens_used": 1626090976,
+      "theoretical_loss": 3.3285856333020103,
+      "tokens_seen": 3255631872
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.7122452254854758e-06,
+      "loss": 1.2244,
+      "theoretical_loss": 3.3285856333020103,
+      "tokens_seen": 3255631872
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.704220831327235e-06,
+      "loss": 1.2673,
+      "theoretical_loss": 3.3285755869521703,
+      "tokens_seen": 3255762944
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.696196437168994e-06,
+      "loss": 1.2142,
+      "theoretical_loss": 3.3285655411200135,
+      "tokens_seen": 3255894016
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.688172043010753e-06,
+      "loss": 1.3027,
+      "theoretical_loss": 3.328555495805493,
+      "tokens_seen": 3256025088
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.680147648852512e-06,
+      "loss": 1.2322,
+      "theoretical_loss": 3.3285454510085604,
+      "tokens_seen": 3256156160
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.672123254694271e-06,
+      "loss": 1.2388,
+      "theoretical_loss": 3.328535406729169,
+      "tokens_seen": 3256287232
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6640988605360297e-06,
+      "loss": 1.2698,
+      "theoretical_loss": 3.328525362967271,
+      "tokens_seen": 3256418304
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6560744663777883e-06,
+      "loss": 1.2942,
+      "theoretical_loss": 3.3285153197228183,
+      "tokens_seen": 3256549376
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6480500722195474e-06,
+      "loss": 1.2482,
+      "theoretical_loss": 3.3285052769957644,
+      "tokens_seen": 3256680448
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6400256780613064e-06,
+      "loss": 1.2399,
+      "theoretical_loss": 3.3284952347860615,
+      "tokens_seen": 3256811520
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6320012839030655e-06,
+      "loss": 1.2579,
+      "theoretical_loss": 3.328485193093662,
+      "tokens_seen": 3256942592
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6239768897448246e-06,
+      "loss": 1.1639,
+      "theoretical_loss": 3.3284751519185183,
+      "tokens_seen": 3257073664
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.615952495586583e-06,
+      "loss": 1.2457,
+      "theoretical_loss": 3.3284651112605834,
+      "tokens_seen": 3257204736
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.6079281014283423e-06,
+      "loss": 1.2198,
+      "theoretical_loss": 3.3284550711198095,
+      "tokens_seen": 3257335808
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.5999037072701013e-06,
+      "loss": 1.2319,
+      "theoretical_loss": 3.328445031496149,
+      "tokens_seen": 3257466880
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.5918793131118604e-06,
+      "loss": 1.2719,
+      "theoretical_loss": 3.328434992389555,
+      "tokens_seen": 3257597952
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.5838549189536194e-06,
+      "loss": 1.1597,
+      "theoretical_loss": 3.3284249537999795,
+      "tokens_seen": 3257729024
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.575830524795378e-06,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3284149157273752,
+      "tokens_seen": 3257860096
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.5678061306371367e-06,
+      "loss": 1.197,
+      "theoretical_loss": 3.3284048781716953,
+      "tokens_seen": 3257991168
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.5597817364788958e-06,
+      "loss": 1.2792,
+      "theoretical_loss": 3.3283948411328916,
+      "tokens_seen": 3258122240
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.551757342320655e-06,
+      "loss": 1.2072,
+      "theoretical_loss": 3.3283848046109172,
+      "tokens_seen": 3258253312
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.543732948162414e-06,
+      "loss": 1.1957,
+      "theoretical_loss": 3.328374768605724,
+      "tokens_seen": 3258384384
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.535708554004173e-06,
+      "loss": 1.2103,
+      "theoretical_loss": 3.328364733117265,
+      "tokens_seen": 3258515456
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 2.5276841598459316e-06,
+      "loss": 1.2648,
+      "theoretical_loss": 3.328354698145494,
+      "tokens_seen": 3258646528
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.5196597656876906e-06,
+      "loss": 1.2299,
+      "theoretical_loss": 3.3283446636903617,
+      "tokens_seen": 3258777600
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.5116353715294497e-06,
+      "loss": 1.1807,
+      "theoretical_loss": 3.3283346297518213,
+      "tokens_seen": 3258908672
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.5036109773712088e-06,
+      "loss": 1.1443,
+      "theoretical_loss": 3.3283245963298262,
+      "tokens_seen": 3259039744
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.495586583212968e-06,
+      "loss": 1.2462,
+      "theoretical_loss": 3.3283145634243283,
+      "tokens_seen": 3259170816
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.4875621890547264e-06,
+      "loss": 1.2156,
+      "theoretical_loss": 3.3283045310352803,
+      "tokens_seen": 3259301888
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.479537794896485e-06,
+      "loss": 1.1577,
+      "theoretical_loss": 3.3282944991626353,
+      "tokens_seen": 3259432960
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.471513400738244e-06,
+      "loss": 1.2757,
+      "theoretical_loss": 3.3282844678063452,
+      "tokens_seen": 3259564032
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.463489006580003e-06,
+      "loss": 1.2126,
+      "theoretical_loss": 3.3282744369663635,
+      "tokens_seen": 3259695104
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.4554646124217623e-06,
+      "loss": 1.2421,
+      "theoretical_loss": 3.328264406642642,
+      "tokens_seen": 3259826176
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.4474402182635213e-06,
+      "loss": 1.2152,
+      "theoretical_loss": 3.3282543768351345,
+      "tokens_seen": 3259957248
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.4394158241052804e-06,
+      "loss": 1.1576,
+      "theoretical_loss": 3.3282443475437926,
+      "tokens_seen": 3260088320
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.431391429947039e-06,
+      "loss": 1.1731,
+      "theoretical_loss": 3.3282343187685695,
+      "tokens_seen": 3260219392
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.423367035788798e-06,
+      "loss": 1.1979,
+      "theoretical_loss": 3.328224290509418,
+      "tokens_seen": 3260350464
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.415342641630557e-06,
+      "loss": 1.1893,
+      "theoretical_loss": 3.32821426276629,
+      "tokens_seen": 3260481536
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.4073182474723158e-06,
+      "loss": 1.1546,
+      "theoretical_loss": 3.3282042355391397,
+      "tokens_seen": 3260612608
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.399293853314075e-06,
+      "loss": 1.1994,
+      "theoretical_loss": 3.3281942088279184,
+      "tokens_seen": 3260743680
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.391269459155834e-06,
+      "loss": 1.1892,
+      "theoretical_loss": 3.328184182632579,
+      "tokens_seen": 3260874752
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3832450649975925e-06,
+      "loss": 1.1361,
+      "theoretical_loss": 3.328174156953075,
+      "tokens_seen": 3261005824
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3752206708393516e-06,
+      "loss": 1.2025,
+      "theoretical_loss": 3.3281641317893587,
+      "tokens_seen": 3261136896
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3671962766811106e-06,
+      "loss": 1.187,
+      "theoretical_loss": 3.3281541071413825,
+      "tokens_seen": 3261267968
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3591718825228697e-06,
+      "loss": 1.2099,
+      "theoretical_loss": 3.3281440830091,
+      "tokens_seen": 3261399040
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3511474883646288e-06,
+      "loss": 1.2226,
+      "theoretical_loss": 3.328134059392463,
+      "tokens_seen": 3261530112
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.343123094206388e-06,
+      "loss": 1.2257,
+      "theoretical_loss": 3.328124036291425,
+      "tokens_seen": 3261661184
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3350987000481465e-06,
+      "loss": 1.2121,
+      "theoretical_loss": 3.328114013705938,
+      "tokens_seen": 3261792256
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3270743058899055e-06,
+      "loss": 1.2138,
+      "theoretical_loss": 3.328103991635955,
+      "tokens_seen": 3261923328
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.319049911731664e-06,
+      "loss": 1.205,
+      "theoretical_loss": 3.3280939700814294,
+      "tokens_seen": 3262054400
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1093665,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1088675260543823,
+      "objective/train/theoretical_loss": 3.3280839490423135,
+      "objective/train/tokens_used": 1632644576,
+      "theoretical_loss": 3.3280839490423135,
+      "tokens_seen": 3262185472
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.311025517573423e-06,
+      "loss": 1.1965,
+      "theoretical_loss": 3.3280839490423135,
+      "tokens_seen": 3262185472
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.3030011234151823e-06,
+      "loss": 1.2071,
+      "theoretical_loss": 3.32807392851856,
+      "tokens_seen": 3262316544
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2949767292569413e-06,
+      "loss": 1.1391,
+      "theoretical_loss": 3.328063908510122,
+      "tokens_seen": 3262447616
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2869523350987e-06,
+      "loss": 1.2993,
+      "theoretical_loss": 3.328053889016952,
+      "tokens_seen": 3262578688
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.278927940940459e-06,
+      "loss": 1.1841,
+      "theoretical_loss": 3.3280438700390027,
+      "tokens_seen": 3262709760
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.270903546782218e-06,
+      "loss": 1.1688,
+      "theoretical_loss": 3.3280338515762278,
+      "tokens_seen": 3262840832
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.262879152623977e-06,
+      "loss": 1.2119,
+      "theoretical_loss": 3.328023833628579,
+      "tokens_seen": 3262971904
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.254854758465736e-06,
+      "loss": 1.1314,
+      "theoretical_loss": 3.3280138161960093,
+      "tokens_seen": 3263102976
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2468303643074953e-06,
+      "loss": 1.1977,
+      "theoretical_loss": 3.328003799278472,
+      "tokens_seen": 3263234048
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.238805970149254e-06,
+      "loss": 1.2158,
+      "theoretical_loss": 3.32799378287592,
+      "tokens_seen": 3263365120
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2307815759910125e-06,
+      "loss": 1.2065,
+      "theoretical_loss": 3.327983766988306,
+      "tokens_seen": 3263496192
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2227571818327716e-06,
+      "loss": 1.1563,
+      "theoretical_loss": 3.3279737516155823,
+      "tokens_seen": 3263627264
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2147327876745306e-06,
+      "loss": 1.1653,
+      "theoretical_loss": 3.327963736757702,
+      "tokens_seen": 3263758336
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.2067083935162897e-06,
+      "loss": 1.2675,
+      "theoretical_loss": 3.3279537224146187,
+      "tokens_seen": 3263889408
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1986839993580488e-06,
+      "loss": 1.2373,
+      "theoretical_loss": 3.327943708586284,
+      "tokens_seen": 3264020480
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1906596051998074e-06,
+      "loss": 1.1878,
+      "theoretical_loss": 3.327933695272652,
+      "tokens_seen": 3264151552
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1826352110415665e-06,
+      "loss": 1.1299,
+      "theoretical_loss": 3.327923682473675,
+      "tokens_seen": 3264282624
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1746108168833255e-06,
+      "loss": 1.1845,
+      "theoretical_loss": 3.327913670189306,
+      "tokens_seen": 3264413696
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1665864227250846e-06,
+      "loss": 1.2085,
+      "theoretical_loss": 3.3279036584194976,
+      "tokens_seen": 3264544768
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.158562028566843e-06,
+      "loss": 1.2402,
+      "theoretical_loss": 3.327893647164203,
+      "tokens_seen": 3264675840
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1505376344086023e-06,
+      "loss": 1.2159,
+      "theoretical_loss": 3.3278836364233753,
+      "tokens_seen": 3264806912
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.142513240250361e-06,
+      "loss": 1.2206,
+      "theoretical_loss": 3.3278736261969666,
+      "tokens_seen": 3264937984
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.13448884609212e-06,
+      "loss": 1.2312,
+      "theoretical_loss": 3.327863616484931,
+      "tokens_seen": 3265069056
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.126464451933879e-06,
+      "loss": 1.2139,
+      "theoretical_loss": 3.3278536072872202,
+      "tokens_seen": 3265200128
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.118440057775638e-06,
+      "loss": 1.2036,
+      "theoretical_loss": 3.327843598603788,
+      "tokens_seen": 3265331200
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.110415663617397e-06,
+      "loss": 1.1893,
+      "theoretical_loss": 3.3278335904345866,
+      "tokens_seen": 3265462272
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.1023912694591558e-06,
+      "loss": 1.2136,
+      "theoretical_loss": 3.32782358277957,
+      "tokens_seen": 3265593344
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.094366875300915e-06,
+      "loss": 1.2325,
+      "theoretical_loss": 3.3278135756386904,
+      "tokens_seen": 3265724416
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.086342481142674e-06,
+      "loss": 1.2093,
+      "theoretical_loss": 3.3278035690119006,
+      "tokens_seen": 3265855488
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.078318086984433e-06,
+      "loss": 1.2544,
+      "theoretical_loss": 3.3277935628991537,
+      "tokens_seen": 3265986560
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0702936928261916e-06,
+      "loss": 1.2256,
+      "theoretical_loss": 3.3277835573004033,
+      "tokens_seen": 3266117632
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0622692986679506e-06,
+      "loss": 1.1718,
+      "theoretical_loss": 3.327773552215602,
+      "tokens_seen": 3266248704
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0542449045097093e-06,
+      "loss": 1.1401,
+      "theoretical_loss": 3.327763547644702,
+      "tokens_seen": 3266379776
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0462205103514683e-06,
+      "loss": 1.2267,
+      "theoretical_loss": 3.327753543587657,
+      "tokens_seen": 3266510848
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0381961161932274e-06,
+      "loss": 1.2306,
+      "theoretical_loss": 3.32774354004442,
+      "tokens_seen": 3266641920
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0301717220349865e-06,
+      "loss": 1.1788,
+      "theoretical_loss": 3.3277335370149443,
+      "tokens_seen": 3266772992
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0221473278767455e-06,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3277235344991825,
+      "tokens_seen": 3266904064
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.0141229337185046e-06,
+      "loss": 1.2312,
+      "theoretical_loss": 3.3277135324970875,
+      "tokens_seen": 3267035136
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.006098539560263e-06,
+      "loss": 1.1506,
+      "theoretical_loss": 3.3277035310086123,
+      "tokens_seen": 3267166208
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.9980741454020223e-06,
+      "loss": 1.1759,
+      "theoretical_loss": 3.3276935300337103,
+      "tokens_seen": 3267297280
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.9900497512437813e-06,
+      "loss": 1.2055,
+      "theoretical_loss": 3.327683529572334,
+      "tokens_seen": 3267428352
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.98202535708554e-06,
+      "loss": 1.215,
+      "theoretical_loss": 3.3276735296244366,
+      "tokens_seen": 3267559424
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.974000962927299e-06,
+      "loss": 1.2129,
+      "theoretical_loss": 3.3276635301899717,
+      "tokens_seen": 3267690496
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.965976568769058e-06,
+      "loss": 1.2298,
+      "theoretical_loss": 3.3276535312688917,
+      "tokens_seen": 3267821568
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.9579521746108167e-06,
+      "loss": 1.2103,
+      "theoretical_loss": 3.32764353286115,
+      "tokens_seen": 3267952640
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.9499277804525758e-06,
+      "loss": 1.2205,
+      "theoretical_loss": 3.3276335349666994,
+      "tokens_seen": 3268083712
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.941903386294335e-06,
+      "loss": 1.2399,
+      "theoretical_loss": 3.327623537585493,
+      "tokens_seen": 3268214784
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.933878992136094e-06,
+      "loss": 1.2509,
+      "theoretical_loss": 3.3276135407174845,
+      "tokens_seen": 3268345856
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.925854597977853e-06,
+      "loss": 1.2128,
+      "theoretical_loss": 3.327603544362626,
+      "tokens_seen": 3268476928
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.917830203819612e-06,
+      "loss": 1.2874,
+      "theoretical_loss": 3.3275935485208707,
+      "tokens_seen": 3268608000
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1095836,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2105580568313599,
+      "objective/train/theoretical_loss": 3.3275835531921722,
+      "objective/train/tokens_used": 1639198176,
+      "theoretical_loss": 3.3275835531921722,
+      "tokens_seen": 3268739072
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.9098058096613706e-06,
+      "loss": 1.2122,
+      "theoretical_loss": 3.3275835531921722,
+      "tokens_seen": 3268739072
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.9017814155031295e-06,
+      "loss": 1.2547,
+      "theoretical_loss": 3.327573558376484,
+      "tokens_seen": 3268870144
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8937570213448886e-06,
+      "loss": 1.2501,
+      "theoretical_loss": 3.327563564073758,
+      "tokens_seen": 3269001216
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8857326271866476e-06,
+      "loss": 1.2647,
+      "theoretical_loss": 3.327553570283948,
+      "tokens_seen": 3269132288
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8777082330284065e-06,
+      "loss": 1.2641,
+      "theoretical_loss": 3.327543577007007,
+      "tokens_seen": 3269263360
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8696838388701655e-06,
+      "loss": 1.256,
+      "theoretical_loss": 3.327533584242888,
+      "tokens_seen": 3269394432
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8616594447119242e-06,
+      "loss": 1.2511,
+      "theoretical_loss": 3.3275235919915445,
+      "tokens_seen": 3269525504
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8536350505536832e-06,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3275136002529297,
+      "tokens_seen": 3269656576
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8456106563954423e-06,
+      "loss": 1.1794,
+      "theoretical_loss": 3.327503609026996,
+      "tokens_seen": 3269787648
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8375862622372011e-06,
+      "loss": 1.1341,
+      "theoretical_loss": 3.327493618313697,
+      "tokens_seen": 3269918720
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8295618680789602e-06,
+      "loss": 1.2023,
+      "theoretical_loss": 3.327483628112986,
+      "tokens_seen": 3270049792
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8215374739207192e-06,
+      "loss": 1.1739,
+      "theoretical_loss": 3.3274736384248156,
+      "tokens_seen": 3270180864
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.8135130797624779e-06,
+      "loss": 1.2157,
+      "theoretical_loss": 3.32746364924914,
+      "tokens_seen": 3270311936
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.805488685604237e-06,
+      "loss": 1.1715,
+      "theoretical_loss": 3.3274536605859115,
+      "tokens_seen": 3270443008
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.797464291445996e-06,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3274436724350833,
+      "tokens_seen": 3270574080
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7894398972877548e-06,
+      "loss": 1.2847,
+      "theoretical_loss": 3.327433684796609,
+      "tokens_seen": 3270705152
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.781415503129514e-06,
+      "loss": 1.2213,
+      "theoretical_loss": 3.327423697670441,
+      "tokens_seen": 3270836224
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.773391108971273e-06,
+      "loss": 1.2229,
+      "theoretical_loss": 3.3274137110565336,
+      "tokens_seen": 3270967296
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7653667148130316e-06,
+      "loss": 1.2241,
+      "theoretical_loss": 3.3274037249548396,
+      "tokens_seen": 3271098368
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7573423206547907e-06,
+      "loss": 1.2302,
+      "theoretical_loss": 3.3273937393653115,
+      "tokens_seen": 3271229440
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7493179264965495e-06,
+      "loss": 1.2632,
+      "theoretical_loss": 3.3273837542879035,
+      "tokens_seen": 3271360512
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7412935323383086e-06,
+      "loss": 1.2366,
+      "theoretical_loss": 3.327373769722568,
+      "tokens_seen": 3271491584
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7332691381800676e-06,
+      "loss": 1.1855,
+      "theoretical_loss": 3.327363785669259,
+      "tokens_seen": 3271622656
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7252447440218267e-06,
+      "loss": 1.247,
+      "theoretical_loss": 3.3273538021279294,
+      "tokens_seen": 3271753728
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7172203498635853e-06,
+      "loss": 1.2417,
+      "theoretical_loss": 3.327343819098532,
+      "tokens_seen": 3271884800
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7091959557053444e-06,
+      "loss": 1.2531,
+      "theoretical_loss": 3.3273338365810208,
+      "tokens_seen": 3272015872
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.7011715615471032e-06,
+      "loss": 1.1733,
+      "theoretical_loss": 3.327323854575349,
+      "tokens_seen": 3272146944
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6931471673888623e-06,
+      "loss": 1.2119,
+      "theoretical_loss": 3.3273138730814686,
+      "tokens_seen": 3272278016
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6851227732306213e-06,
+      "loss": 1.2826,
+      "theoretical_loss": 3.3273038920993345,
+      "tokens_seen": 3272409088
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.67709837907238e-06,
+      "loss": 1.2076,
+      "theoretical_loss": 3.327293911628899,
+      "tokens_seen": 3272540160
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.669073984914139e-06,
+      "loss": 1.1623,
+      "theoretical_loss": 3.3272839316701157,
+      "tokens_seen": 3272671232
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6610495907558979e-06,
+      "loss": 1.228,
+      "theoretical_loss": 3.3272739522229378,
+      "tokens_seen": 3272802304
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.653025196597657e-06,
+      "loss": 1.2491,
+      "theoretical_loss": 3.3272639732873186,
+      "tokens_seen": 3272933376
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.645000802439416e-06,
+      "loss": 1.1942,
+      "theoretical_loss": 3.3272539948632116,
+      "tokens_seen": 3273064448
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.636976408281175e-06,
+      "loss": 1.195,
+      "theoretical_loss": 3.3272440169505697,
+      "tokens_seen": 3273195520
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6289520141229337e-06,
+      "loss": 1.1692,
+      "theoretical_loss": 3.3272340395493463,
+      "tokens_seen": 3273326592
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6209276199646925e-06,
+      "loss": 1.2258,
+      "theoretical_loss": 3.3272240626594947,
+      "tokens_seen": 3273457664
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6129032258064516e-06,
+      "loss": 1.2096,
+      "theoretical_loss": 3.3272140862809687,
+      "tokens_seen": 3273588736
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.6048788316482107e-06,
+      "loss": 1.1739,
+      "theoretical_loss": 3.327204110413721,
+      "tokens_seen": 3273719808
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5968544374899697e-06,
+      "loss": 1.256,
+      "theoretical_loss": 3.327194135057705,
+      "tokens_seen": 3273850880
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5888300433317286e-06,
+      "loss": 1.2477,
+      "theoretical_loss": 3.3271841602128744,
+      "tokens_seen": 3273981952
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5808056491734874e-06,
+      "loss": 1.2838,
+      "theoretical_loss": 3.3271741858791826,
+      "tokens_seen": 3274113024
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5727812550152463e-06,
+      "loss": 1.2226,
+      "theoretical_loss": 3.3271642120565827,
+      "tokens_seen": 3274244096
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5647568608570053e-06,
+      "loss": 1.1582,
+      "theoretical_loss": 3.3271542387450275,
+      "tokens_seen": 3274375168
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5567324666987644e-06,
+      "loss": 1.2565,
+      "theoretical_loss": 3.3271442659444714,
+      "tokens_seen": 3274506240
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5487080725405234e-06,
+      "loss": 1.2546,
+      "theoretical_loss": 3.327134293654867,
+      "tokens_seen": 3274637312
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5406836783822823e-06,
+      "loss": 1.1838,
+      "theoretical_loss": 3.327124321876168,
+      "tokens_seen": 3274768384
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5326592842240411e-06,
+      "loss": 1.2096,
+      "theoretical_loss": 3.327114350608328,
+      "tokens_seen": 3274899456
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.5246348900658002e-06,
+      "loss": 1.2291,
+      "theoretical_loss": 3.3271043798513,
+      "tokens_seen": 3275030528
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.516610495907559e-06,
+      "loss": 1.2042,
+      "theoretical_loss": 3.327094409605037,
+      "tokens_seen": 3275161600
+    },
+    {
+      "epoch": 0.98,
+      "objective/train/docs_used": 1097820,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1394554376602173,
+      "objective/train/theoretical_loss": 3.3270844398694934,
+      "objective/train/tokens_used": 1645751776,
+      "theoretical_loss": 3.3270844398694934,
+      "tokens_seen": 3275292672
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.508586101749318e-06,
+      "loss": 1.1548,
+      "theoretical_loss": 3.3270844398694934,
+      "tokens_seen": 3275292672
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.500561707591077e-06,
+      "loss": 1.2728,
+      "theoretical_loss": 3.327074470644622,
+      "tokens_seen": 3275423744
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4925373134328358e-06,
+      "loss": 1.2834,
+      "theoretical_loss": 3.3270645019303764,
+      "tokens_seen": 3275554816
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4845129192745948e-06,
+      "loss": 1.1998,
+      "theoretical_loss": 3.32705453372671,
+      "tokens_seen": 3275685888
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4764885251163537e-06,
+      "loss": 1.1975,
+      "theoretical_loss": 3.3270445660335755,
+      "tokens_seen": 3275816960
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4684641309581128e-06,
+      "loss": 1.2389,
+      "theoretical_loss": 3.3270345988509273,
+      "tokens_seen": 3275948032
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4604397367998718e-06,
+      "loss": 1.2306,
+      "theoretical_loss": 3.327024632178719,
+      "tokens_seen": 3276079104
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4524153426416304e-06,
+      "loss": 1.2276,
+      "theoretical_loss": 3.327014666016903,
+      "tokens_seen": 3276210176
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4443909484833895e-06,
+      "loss": 1.2164,
+      "theoretical_loss": 3.3270047003654337,
+      "tokens_seen": 3276341248
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4363665543251486e-06,
+      "loss": 1.1549,
+      "theoretical_loss": 3.3269947352242637,
+      "tokens_seen": 3276472320
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4283421601669074e-06,
+      "loss": 1.2414,
+      "theoretical_loss": 3.326984770593347,
+      "tokens_seen": 3276603392
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4203177660086665e-06,
+      "loss": 1.2279,
+      "theoretical_loss": 3.326974806472637,
+      "tokens_seen": 3276734464
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4122933718504253e-06,
+      "loss": 1.2207,
+      "theoretical_loss": 3.326964842862087,
+      "tokens_seen": 3276865536
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4042689776921842e-06,
+      "loss": 1.1802,
+      "theoretical_loss": 3.326954879761651,
+      "tokens_seen": 3276996608
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3962445835339432e-06,
+      "loss": 1.1895,
+      "theoretical_loss": 3.3269449171712817,
+      "tokens_seen": 3277127680
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3882201893757023e-06,
+      "loss": 1.1755,
+      "theoretical_loss": 3.326934955090933,
+      "tokens_seen": 3277258752
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3801957952174611e-06,
+      "loss": 1.2468,
+      "theoretical_loss": 3.3269249935205583,
+      "tokens_seen": 3277389824
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3721714010592202e-06,
+      "loss": 1.2162,
+      "theoretical_loss": 3.3269150324601116,
+      "tokens_seen": 3277520896
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.364147006900979e-06,
+      "loss": 1.3013,
+      "theoretical_loss": 3.3269050719095454,
+      "tokens_seen": 3277651968
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3561226127427379e-06,
+      "loss": 1.2333,
+      "theoretical_loss": 3.326895111868814,
+      "tokens_seen": 3277783040
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.348098218584497e-06,
+      "loss": 1.2561,
+      "theoretical_loss": 3.326885152337871,
+      "tokens_seen": 3277914112
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.340073824426256e-06,
+      "loss": 1.2558,
+      "theoretical_loss": 3.326875193316669,
+      "tokens_seen": 3278045184
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3320494302680148e-06,
+      "loss": 1.2257,
+      "theoretical_loss": 3.3268652348051626,
+      "tokens_seen": 3278176256
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3240250361097737e-06,
+      "loss": 1.3222,
+      "theoretical_loss": 3.326855276803305,
+      "tokens_seen": 3278307328
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3160006419515328e-06,
+      "loss": 1.2467,
+      "theoretical_loss": 3.326845319311049,
+      "tokens_seen": 3278438400
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3079762477932916e-06,
+      "loss": 1.2203,
+      "theoretical_loss": 3.3268353623283495,
+      "tokens_seen": 3278569472
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2999518536350507e-06,
+      "loss": 1.2421,
+      "theoretical_loss": 3.326825405855159,
+      "tokens_seen": 3278700544
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2919274594768097e-06,
+      "loss": 1.3106,
+      "theoretical_loss": 3.3268154498914315,
+      "tokens_seen": 3278831616
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2839030653185684e-06,
+      "loss": 1.259,
+      "theoretical_loss": 3.326805494437121,
+      "tokens_seen": 3278962688
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2758786711603274e-06,
+      "loss": 1.1572,
+      "theoretical_loss": 3.3267955394921795,
+      "tokens_seen": 3279093760
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2678542770020865e-06,
+      "loss": 1.2932,
+      "theoretical_loss": 3.3267855850565624,
+      "tokens_seen": 3279224832
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2598298828438453e-06,
+      "loss": 1.22,
+      "theoretical_loss": 3.326775631130222,
+      "tokens_seen": 3279355904
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2518054886856044e-06,
+      "loss": 1.2011,
+      "theoretical_loss": 3.3267656777131123,
+      "tokens_seen": 3279486976
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2437810945273632e-06,
+      "loss": 1.2292,
+      "theoretical_loss": 3.3267557248051873,
+      "tokens_seen": 3279618048
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.235756700369122e-06,
+      "loss": 1.2163,
+      "theoretical_loss": 3.3267457724064005,
+      "tokens_seen": 3279749120
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2277323062108811e-06,
+      "loss": 1.2127,
+      "theoretical_loss": 3.326735820516705,
+      "tokens_seen": 3279880192
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2197079120526402e-06,
+      "loss": 1.1797,
+      "theoretical_loss": 3.326725869136055,
+      "tokens_seen": 3280011264
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.211683517894399e-06,
+      "loss": 1.2057,
+      "theoretical_loss": 3.3267159182644033,
+      "tokens_seen": 3280142336
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.2036591237361579e-06,
+      "loss": 1.3442,
+      "theoretical_loss": 3.326705967901704,
+      "tokens_seen": 3280273408
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.195634729577917e-06,
+      "loss": 1.2546,
+      "theoretical_loss": 3.3266960180479117,
+      "tokens_seen": 3280404480
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1876103354196758e-06,
+      "loss": 1.2806,
+      "theoretical_loss": 3.3266860687029784,
+      "tokens_seen": 3280535552
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1795859412614348e-06,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3266761198668586,
+      "tokens_seen": 3280666624
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.171561547103194e-06,
+      "loss": 1.2594,
+      "theoretical_loss": 3.3266661715395065,
+      "tokens_seen": 3280797696
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1635371529449528e-06,
+      "loss": 1.241,
+      "theoretical_loss": 3.326656223720874,
+      "tokens_seen": 3280928768
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1555127587867116e-06,
+      "loss": 1.2027,
+      "theoretical_loss": 3.3266462764109166,
+      "tokens_seen": 3281059840
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1474883646284707e-06,
+      "loss": 1.193,
+      "theoretical_loss": 3.3266363296095873,
+      "tokens_seen": 3281190912
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1394639704702295e-06,
+      "loss": 1.229,
+      "theoretical_loss": 3.3266263833168392,
+      "tokens_seen": 3281321984
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1314395763119886e-06,
+      "loss": 1.2525,
+      "theoretical_loss": 3.326616437532627,
+      "tokens_seen": 3281453056
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1234151821537476e-06,
+      "loss": 1.2046,
+      "theoretical_loss": 3.3266064922569036,
+      "tokens_seen": 3281584128
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1153907879955063e-06,
+      "loss": 1.2835,
+      "theoretical_loss": 3.326596547489623,
+      "tokens_seen": 3281715200
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/docs_used": 1099909,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1573256254196167,
+      "objective/train/theoretical_loss": 3.326586603230739,
+      "objective/train/tokens_used": 1652305376,
+      "theoretical_loss": 3.326586603230739,
+      "tokens_seen": 3281846272
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1073663938372653e-06,
+      "loss": 1.2082,
+      "theoretical_loss": 3.326586603230739,
+      "tokens_seen": 3281846272
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0993419996790244e-06,
+      "loss": 1.1748,
+      "theoretical_loss": 3.326576659480205,
+      "tokens_seen": 3281977344
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0913176055207832e-06,
+      "loss": 1.1914,
+      "theoretical_loss": 3.326566716237975,
+      "tokens_seen": 3282108416
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0832932113625423e-06,
+      "loss": 1.225,
+      "theoretical_loss": 3.3265567735040023,
+      "tokens_seen": 3282239488
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0752688172043011e-06,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3265468312782414,
+      "tokens_seen": 3282370560
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.06724442304606e-06,
+      "loss": 1.2002,
+      "theoretical_loss": 3.3265368895606455,
+      "tokens_seen": 3282501632
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.059220028887819e-06,
+      "loss": 1.2121,
+      "theoretical_loss": 3.326526948351168,
+      "tokens_seen": 3282632704
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0511956347295779e-06,
+      "loss": 1.3033,
+      "theoretical_loss": 3.3265170076497634,
+      "tokens_seen": 3282763776
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.043171240571337e-06,
+      "loss": 1.2792,
+      "theoretical_loss": 3.326507067456385,
+      "tokens_seen": 3282894848
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0351468464130958e-06,
+      "loss": 1.2685,
+      "theoretical_loss": 3.3264971277709865,
+      "tokens_seen": 3283025920
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0271224522548546e-06,
+      "loss": 1.2682,
+      "theoretical_loss": 3.3264871885935223,
+      "tokens_seen": 3283156992
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0190980580966137e-06,
+      "loss": 1.195,
+      "theoretical_loss": 3.3264772499239452,
+      "tokens_seen": 3283288064
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0110736639383728e-06,
+      "loss": 1.2208,
+      "theoretical_loss": 3.32646731176221,
+      "tokens_seen": 3283419136
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0030492697801316e-06,
+      "loss": 1.2856,
+      "theoretical_loss": 3.3264573741082692,
+      "tokens_seen": 3283550208
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.950248756218907e-07,
+      "loss": 1.201,
+      "theoretical_loss": 3.3264474369620776,
+      "tokens_seen": 3283681280
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.870004814636495e-07,
+      "loss": 1.2444,
+      "theoretical_loss": 3.326437500323589,
+      "tokens_seen": 3283812352
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.789760873054084e-07,
+      "loss": 1.2333,
+      "theoretical_loss": 3.3264275641927568,
+      "tokens_seen": 3283943424
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.709516931471674e-07,
+      "loss": 1.1843,
+      "theoretical_loss": 3.3264176285695344,
+      "tokens_seen": 3284074496
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.629272989889265e-07,
+      "loss": 1.1811,
+      "theoretical_loss": 3.3264076934538767,
+      "tokens_seen": 3284205568
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.549029048306853e-07,
+      "loss": 1.2512,
+      "theoretical_loss": 3.326397758845737,
+      "tokens_seen": 3284336640
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.468785106724443e-07,
+      "loss": 1.2137,
+      "theoretical_loss": 3.3263878247450687,
+      "tokens_seen": 3284467712
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.388541165142032e-07,
+      "loss": 1.2402,
+      "theoretical_loss": 3.326377891151826,
+      "tokens_seen": 3284598784
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.308297223559621e-07,
+      "loss": 1.1966,
+      "theoretical_loss": 3.326367958065963,
+      "tokens_seen": 3284729856
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.228053281977211e-07,
+      "loss": 1.26,
+      "theoretical_loss": 3.326358025487433,
+      "tokens_seen": 3284860928
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.147809340394801e-07,
+      "loss": 1.1888,
+      "theoretical_loss": 3.3263480934161906,
+      "tokens_seen": 3284992000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 9.067565398812389e-07,
+      "loss": 1.322,
+      "theoretical_loss": 3.326338161852189,
+      "tokens_seen": 3285123072
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.98732145722998e-07,
+      "loss": 1.2055,
+      "theoretical_loss": 3.3263282307953816,
+      "tokens_seen": 3285254144
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.90707751564757e-07,
+      "loss": 1.2946,
+      "theoretical_loss": 3.3263183002457235,
+      "tokens_seen": 3285385216
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.826833574065158e-07,
+      "loss": 1.2426,
+      "theoretical_loss": 3.3263083702031677,
+      "tokens_seen": 3285516288
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.746589632482747e-07,
+      "loss": 1.311,
+      "theoretical_loss": 3.326298440667668,
+      "tokens_seen": 3285647360
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.666345690900338e-07,
+      "loss": 1.2349,
+      "theoretical_loss": 3.326288511639179,
+      "tokens_seen": 3285778432
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.586101749317927e-07,
+      "loss": 1.255,
+      "theoretical_loss": 3.3262785831176545,
+      "tokens_seen": 3285909504
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.505857807735516e-07,
+      "loss": 1.1904,
+      "theoretical_loss": 3.3262686551030476,
+      "tokens_seen": 3286040576
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.425613866153107e-07,
+      "loss": 1.2299,
+      "theoretical_loss": 3.326258727595313,
+      "tokens_seen": 3286171648
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.345369924570695e-07,
+      "loss": 1.2286,
+      "theoretical_loss": 3.326248800594404,
+      "tokens_seen": 3286302720
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.265125982988285e-07,
+      "loss": 1.2753,
+      "theoretical_loss": 3.326238874100275,
+      "tokens_seen": 3286433792
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.184882041405875e-07,
+      "loss": 1.2835,
+      "theoretical_loss": 3.3262289481128797,
+      "tokens_seen": 3286564864
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.104638099823463e-07,
+      "loss": 1.2156,
+      "theoretical_loss": 3.326219022632172,
+      "tokens_seen": 3286695936
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.024394158241053e-07,
+      "loss": 1.2539,
+      "theoretical_loss": 3.326209097658106,
+      "tokens_seen": 3286827008
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.944150216658643e-07,
+      "loss": 1.251,
+      "theoretical_loss": 3.3261991731906355,
+      "tokens_seen": 3286958080
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.863906275076231e-07,
+      "loss": 1.2413,
+      "theoretical_loss": 3.3261892492297145,
+      "tokens_seen": 3287089152
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.783662333493822e-07,
+      "loss": 1.223,
+      "theoretical_loss": 3.3261793257752967,
+      "tokens_seen": 3287220224
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.703418391911411e-07,
+      "loss": 1.2079,
+      "theoretical_loss": 3.3261694028273365,
+      "tokens_seen": 3287351296
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.623174450329001e-07,
+      "loss": 1.3016,
+      "theoretical_loss": 3.3261594803857877,
+      "tokens_seen": 3287482368
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.54293050874659e-07,
+      "loss": 1.3532,
+      "theoretical_loss": 3.326149558450604,
+      "tokens_seen": 3287613440
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.462686567164179e-07,
+      "loss": 1.2668,
+      "theoretical_loss": 3.3261396370217393,
+      "tokens_seen": 3287744512
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.382442625581768e-07,
+      "loss": 1.2741,
+      "theoretical_loss": 3.3261297160991483,
+      "tokens_seen": 3287875584
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.302198683999359e-07,
+      "loss": 1.2664,
+      "theoretical_loss": 3.3261197956827844,
+      "tokens_seen": 3288006656
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.221954742416948e-07,
+      "loss": 1.2441,
+      "theoretical_loss": 3.3261098757726018,
+      "tokens_seen": 3288137728
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.141710800834537e-07,
+      "loss": 1.2348,
+      "theoretical_loss": 3.3260999563685543,
+      "tokens_seen": 3288268800
+    },
+    {
+      "epoch": 0.99,
+      "objective/train/docs_used": 1102163,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.2218878269195557,
+      "objective/train/theoretical_loss": 3.3260900374705957,
+      "objective/train/tokens_used": 1658858976,
+      "theoretical_loss": 3.3260900374705957,
+      "tokens_seen": 3288399872
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.061466859252127e-07,
+      "loss": 1.241,
+      "theoretical_loss": 3.3260900374705957,
+      "tokens_seen": 3288399872
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.981222917669716e-07,
+      "loss": 1.278,
+      "theoretical_loss": 3.326080119078681,
+      "tokens_seen": 3288530944
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.900978976087306e-07,
+      "loss": 1.2473,
+      "theoretical_loss": 3.326070201192763,
+      "tokens_seen": 3288662016
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.820735034504895e-07,
+      "loss": 1.1628,
+      "theoretical_loss": 3.3260602838127964,
+      "tokens_seen": 3288793088
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.740491092922485e-07,
+      "loss": 1.2752,
+      "theoretical_loss": 3.3260503669387353,
+      "tokens_seen": 3288924160
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.660247151340074e-07,
+      "loss": 1.258,
+      "theoretical_loss": 3.326040450570533,
+      "tokens_seen": 3289055232
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.580003209757664e-07,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3260305347081447,
+      "tokens_seen": 3289186304
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.499759268175253e-07,
+      "loss": 1.1856,
+      "theoretical_loss": 3.3260206193515236,
+      "tokens_seen": 3289317376
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.419515326592842e-07,
+      "loss": 1.2694,
+      "theoretical_loss": 3.3260107045006237,
+      "tokens_seen": 3289448448
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.339271385010432e-07,
+      "loss": 1.2269,
+      "theoretical_loss": 3.3260007901554,
+      "tokens_seen": 3289579520
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.259027443428022e-07,
+      "loss": 1.1608,
+      "theoretical_loss": 3.3259908763158053,
+      "tokens_seen": 3289710592
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.17878350184561e-07,
+      "loss": 1.2478,
+      "theoretical_loss": 3.3259809629817942,
+      "tokens_seen": 3289841664
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.098539560263201e-07,
+      "loss": 1.2368,
+      "theoretical_loss": 3.325971050153321,
+      "tokens_seen": 3289972736
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 6.018295618680789e-07,
+      "loss": 1.2799,
+      "theoretical_loss": 3.3259611378303395,
+      "tokens_seen": 3290103808
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.938051677098379e-07,
+      "loss": 1.2346,
+      "theoretical_loss": 3.325951226012804,
+      "tokens_seen": 3290234880
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.85780773551597e-07,
+      "loss": 1.2503,
+      "theoretical_loss": 3.3259413147006684,
+      "tokens_seen": 3290365952
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.777563793933558e-07,
+      "loss": 1.2865,
+      "theoretical_loss": 3.325931403893887,
+      "tokens_seen": 3290497024
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.697319852351148e-07,
+      "loss": 1.2888,
+      "theoretical_loss": 3.3259214935924133,
+      "tokens_seen": 3290628096
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.617075910768738e-07,
+      "loss": 1.1945,
+      "theoretical_loss": 3.3259115837962026,
+      "tokens_seen": 3290759168
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.536831969186327e-07,
+      "loss": 1.1632,
+      "theoretical_loss": 3.3259016745052077,
+      "tokens_seen": 3290890240
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.456588027603916e-07,
+      "loss": 1.2452,
+      "theoretical_loss": 3.325891765719384,
+      "tokens_seen": 3291021312
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.376344086021506e-07,
+      "loss": 1.2143,
+      "theoretical_loss": 3.3258818574386844,
+      "tokens_seen": 3291152384
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.296100144439095e-07,
+      "loss": 1.1984,
+      "theoretical_loss": 3.325871949663064,
+      "tokens_seen": 3291283456
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.215856202856685e-07,
+      "loss": 1.2983,
+      "theoretical_loss": 3.325862042392476,
+      "tokens_seen": 3291414528
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.135612261274273e-07,
+      "loss": 1.3051,
+      "theoretical_loss": 3.3258521356268753,
+      "tokens_seen": 3291545600
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 5.055368319691864e-07,
+      "loss": 1.2283,
+      "theoretical_loss": 3.325842229366216,
+      "tokens_seen": 3291676672
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.975124378109453e-07,
+      "loss": 1.2356,
+      "theoretical_loss": 3.3258323236104523,
+      "tokens_seen": 3291807744
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.894880436527042e-07,
+      "loss": 1.2496,
+      "theoretical_loss": 3.3258224183595377,
+      "tokens_seen": 3291938816
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.814636494944632e-07,
+      "loss": 1.2534,
+      "theoretical_loss": 3.325812513613427,
+      "tokens_seen": 3292069888
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.7343925533622214e-07,
+      "loss": 1.2712,
+      "theoretical_loss": 3.325802609372074,
+      "tokens_seen": 3292200960
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.6541486117798104e-07,
+      "loss": 1.2617,
+      "theoretical_loss": 3.3257927056354335,
+      "tokens_seen": 3292332032
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.5739046701974004e-07,
+      "loss": 1.2357,
+      "theoretical_loss": 3.325782802403459,
+      "tokens_seen": 3292463104
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.49366072861499e-07,
+      "loss": 1.2521,
+      "theoretical_loss": 3.3257728996761053,
+      "tokens_seen": 3292594176
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.413416787032579e-07,
+      "loss": 1.2887,
+      "theoretical_loss": 3.325762997453326,
+      "tokens_seen": 3292725248
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.333172845450169e-07,
+      "loss": 1.218,
+      "theoretical_loss": 3.3257530957350756,
+      "tokens_seen": 3292856320
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.252928903867758e-07,
+      "loss": 1.2816,
+      "theoretical_loss": 3.325743194521308,
+      "tokens_seen": 3292987392
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.1726849622853476e-07,
+      "loss": 1.2676,
+      "theoretical_loss": 3.3257332938119784,
+      "tokens_seen": 3293118464
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.0924410207029376e-07,
+      "loss": 1.2045,
+      "theoretical_loss": 3.32572339360704,
+      "tokens_seen": 3293249536
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.0121970791205266e-07,
+      "loss": 1.2548,
+      "theoretical_loss": 3.325713493906447,
+      "tokens_seen": 3293380608
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.9319531375381156e-07,
+      "loss": 1.2447,
+      "theoretical_loss": 3.3257035947101543,
+      "tokens_seen": 3293511680
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.8517091959557057e-07,
+      "loss": 1.2431,
+      "theoretical_loss": 3.3256936960181163,
+      "tokens_seen": 3293642752
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.771465254373295e-07,
+      "loss": 1.2078,
+      "theoretical_loss": 3.325683797830286,
+      "tokens_seen": 3293773824
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.691221312790884e-07,
+      "loss": 1.3146,
+      "theoretical_loss": 3.325673900146619,
+      "tokens_seen": 3293904896
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.610977371208474e-07,
+      "loss": 1.1881,
+      "theoretical_loss": 3.325664002967069,
+      "tokens_seen": 3294035968
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.5307334296260633e-07,
+      "loss": 1.2403,
+      "theoretical_loss": 3.32565410629159,
+      "tokens_seen": 3294167040
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.450489488043653e-07,
+      "loss": 1.2385,
+      "theoretical_loss": 3.3256442101201364,
+      "tokens_seen": 3294298112
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.3702455464612424e-07,
+      "loss": 1.3217,
+      "theoretical_loss": 3.325634314452663,
+      "tokens_seen": 3294429184
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.290001604878832e-07,
+      "loss": 1.2418,
+      "theoretical_loss": 3.325624419289124,
+      "tokens_seen": 3294560256
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.209757663296421e-07,
+      "loss": 1.2498,
+      "theoretical_loss": 3.3256145246294726,
+      "tokens_seen": 3294691328
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.129513721714011e-07,
+      "loss": 1.3143,
+      "theoretical_loss": 3.325604630473664,
+      "tokens_seen": 3294822400
+    },
+    {
+      "epoch": 1.0,
+      "objective/train/docs_used": 1104299,
+      "objective/train/instantaneous_batch_size": 64,
+      "objective/train/instantaneous_microbatch_size": 65536,
+      "objective/train/original_loss": 1.1802502870559692,
+      "objective/train/theoretical_loss": 3.325594736821653,
+      "objective/train/tokens_used": 1665412576,
+      "theoretical_loss": 3.325594736821653,
+      "tokens_seen": 3294953472
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.0492697801316005e-07,
+      "loss": 1.1573,
+      "theoretical_loss": 3.325594736821653,
+      "tokens_seen": 3294953472
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.9690258385491895e-07,
+      "loss": 1.2497,
+      "theoretical_loss": 3.325584843673393,
+      "tokens_seen": 3295084544
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.888781896966779e-07,
+      "loss": 1.1869,
+      "theoretical_loss": 3.3255749510288384,
+      "tokens_seen": 3295215616
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.808537955384369e-07,
+      "loss": 1.2554,
+      "theoretical_loss": 3.325565058887944,
+      "tokens_seen": 3295346688
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.728294013801958e-07,
+      "loss": 1.2263,
+      "theoretical_loss": 3.3255551672506636,
+      "tokens_seen": 3295477760
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.6480500722195476e-07,
+      "loss": 1.2218,
+      "theoretical_loss": 3.3255452761169524,
+      "tokens_seen": 3295608832
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.5678061306371366e-07,
+      "loss": 1.2004,
+      "theoretical_loss": 3.3255353854867633,
+      "tokens_seen": 3295739904
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.4875621890547267e-07,
+      "loss": 1.2329,
+      "theoretical_loss": 3.325525495360052,
+      "tokens_seen": 3295870976
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.407318247472316e-07,
+      "loss": 1.3252,
+      "theoretical_loss": 3.325515605736772,
+      "tokens_seen": 3296002048
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.3270743058899052e-07,
+      "loss": 1.2337,
+      "theoretical_loss": 3.3255057166168784,
+      "tokens_seen": 3296133120
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.246830364307495e-07,
+      "loss": 1.1715,
+      "theoretical_loss": 3.3254958280003253,
+      "tokens_seen": 3296264192
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.1665864227250845e-07,
+      "loss": 1.2219,
+      "theoretical_loss": 3.3254859398870664,
+      "tokens_seen": 3296395264
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.0863424811426738e-07,
+      "loss": 1.1948,
+      "theoretical_loss": 3.3254760522770566,
+      "tokens_seen": 3296526336
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.0060985395602633e-07,
+      "loss": 1.1993,
+      "theoretical_loss": 3.3254661651702504,
+      "tokens_seen": 3296657408
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.9258545979778528e-07,
+      "loss": 1.2684,
+      "theoretical_loss": 3.325456278566602,
+      "tokens_seen": 3296788480
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.845610656395442e-07,
+      "loss": 1.261,
+      "theoretical_loss": 3.325446392466066,
+      "tokens_seen": 3296919552
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.7653667148130316e-07,
+      "loss": 1.2195,
+      "theoretical_loss": 3.3254365068685967,
+      "tokens_seen": 3297050624
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6851227732306212e-07,
+      "loss": 1.2345,
+      "theoretical_loss": 3.3254266217741484,
+      "tokens_seen": 3297181696
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6048788316482104e-07,
+      "loss": 1.2217,
+      "theoretical_loss": 3.3254167371826755,
+      "tokens_seen": 3297312768
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.5246348900658002e-07,
+      "loss": 1.246,
+      "theoretical_loss": 3.3254068530941323,
+      "tokens_seen": 3297443840
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.4443909484833895e-07,
+      "loss": 1.1599,
+      "theoretical_loss": 3.3253969695084735,
+      "tokens_seen": 3297574912
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.364147006900979e-07,
+      "loss": 1.3043,
+      "theoretical_loss": 3.3253870864256534,
+      "tokens_seen": 3297705984
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2839030653185683e-07,
+      "loss": 1.2615,
+      "theoretical_loss": 3.3253772038456266,
+      "tokens_seen": 3297837056
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.203659123736158e-07,
+      "loss": 1.2059,
+      "theoretical_loss": 3.325367321768347,
+      "tokens_seen": 3297968128
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.1234151821537475e-07,
+      "loss": 1.2567,
+      "theoretical_loss": 3.3253574401937698,
+      "tokens_seen": 3298099200
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0431712405713369e-07,
+      "loss": 1.1947,
+      "theoretical_loss": 3.325347559121849,
+      "tokens_seen": 3298230272
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 9.629272989889264e-08,
+      "loss": 1.2833,
+      "theoretical_loss": 3.3253376785525393,
+      "tokens_seen": 3298361344
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.826833574065158e-08,
+      "loss": 1.288,
+      "theoretical_loss": 3.3253277984857945,
+      "tokens_seen": 3298492416
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.024394158241052e-08,
+      "loss": 1.2176,
+      "theoretical_loss": 3.32531791892157,
+      "tokens_seen": 3298623488
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 7.221954742416948e-08,
+      "loss": 1.2547,
+      "theoretical_loss": 3.32530803985982,
+      "tokens_seen": 3298754560
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 6.419515326592842e-08,
+      "loss": 1.1789,
+      "theoretical_loss": 3.3252981613004984,
+      "tokens_seen": 3298885632
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 5.6170759107687375e-08,
+      "loss": 1.1824,
+      "theoretical_loss": 3.32528828324356,
+      "tokens_seen": 3299016704
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.814636494944632e-08,
+      "loss": 1.211,
+      "theoretical_loss": 3.3252784056889597,
+      "tokens_seen": 3299147776
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.012197079120526e-08,
+      "loss": 1.1891,
+      "theoretical_loss": 3.3252685286366517,
+      "tokens_seen": 3299278848
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.209757663296421e-08,
+      "loss": 1.2315,
+      "theoretical_loss": 3.3252586520865908,
+      "tokens_seen": 3299409920
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.407318247472316e-08,
+      "loss": 1.1526,
+      "theoretical_loss": 3.325248776038731,
+      "tokens_seen": 3299540992
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.6048788316482104e-08,
+      "loss": 1.2474,
+      "theoretical_loss": 3.3252389004930265,
+      "tokens_seen": 3299672064
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.024394158241052e-09,
+      "loss": 1.2109,
+      "theoretical_loss": 3.325229025449433,
+      "tokens_seen": 3299803136
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 0.0,
+      "loss": 1.2071,
+      "theoretical_loss": 3.325219150907904,
+      "tokens_seen": 3299934208
+    }
+  ],
+  "max_steps": 12588,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 8.42020889296896e+17,
+  "trial_name": null,
+  "trial_params": null
+}